From 8522b26c682d566743c7439a046afd99f19ab238 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 16 Sep 2025 10:31:18 -0700 Subject: [PATCH 1/7] m --- test-server/Makefile | 33 +- test-server/go-server/.gitignore | 48 +++ test-server/go-server/Makefile | 69 +++ test-server/go-server/README.md | 148 +++++++ test-server/go-server/go.mod | 31 ++ test-server/go-server/go.sum | 46 ++ test-server/go-server/main.go | 395 ++++++++++++++++++ .../amazon/encryption/s3/RoundTripTests.java | 77 +++- 8 files changed, 833 insertions(+), 14 deletions(-) create mode 100644 test-server/go-server/.gitignore create mode 100644 test-server/go-server/Makefile create mode 100644 test-server/go-server/README.md create mode 100644 test-server/go-server/go.mod create mode 100644 test-server/go-server/go.sum create mode 100644 test-server/go-server/main.go diff --git a/test-server/Makefile b/test-server/Makefile index afbe97b5..60326393 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -1,6 +1,6 @@ # Makefile for S3 Encryption Client Testing -.PHONY: all start-servers start-python-server start-java-server run-tests stop-servers clean ci check-env help +.PHONY: all start-servers start-python-server start-java-server start-go-server run-tests stop-servers clean ci check-env help # Default target all: start-servers run-tests @@ -35,16 +35,28 @@ start-java-server: ./gradlew --build-cache --parallel run & echo $$! > ../java-server.pid @echo "Java server starting..." -# Start both servers in parallel +# Start Go server in background +start-go-server: + @echo "Starting Go server..." + cd go-server && \ + go mod tidy && \ + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + go run . & echo $$! > ../go-server.pid + @echo "Go server starting..." + +# Start all servers in parallel start-servers: @echo "Starting servers in parallel..." - @$(MAKE) -j2 start-python-server start-java-server + @$(MAKE) -j3 start-python-server start-java-server start-go-server @echo "Waiting for servers to be ready..." @for i in $$(seq 1 360); do \ - if nc -z localhost 8080 && nc -z localhost 8081; then \ + if nc -z localhost 8080 && nc -z localhost 8081 && nc -z localhost 8082; then \ echo "Ports are open, waiting for servers to initialize..."; \ sleep 5; \ - echo "Both servers are ready!"; \ + echo "All servers are ready!"; \ break; \ fi; \ if [ $$i -eq 360 ]; then \ @@ -66,7 +78,7 @@ run-tests: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./gradlew --build-cache --parallel integ + ./gradlew --build-cache --parallel -i integ @echo "Tests completed successfully" # Stop the servers @@ -80,12 +92,16 @@ stop-servers: kill $$(cat java-server.pid) 2>/dev/null || true; \ rm java-server.pid; \ fi + @if [ -f go-server.pid ]; then \ + kill $$(cat go-server.pid) 2>/dev/null || true; \ + rm go-server.pid; \ + fi @echo "Servers stopped" # Clean up logs and pid files clean: stop-servers @echo "Cleaning up..." - @rm -f python-server.log java-server.log + @rm -f python-server.log java-server.log go-server.log @echo "Cleanup complete" # Help target @@ -93,9 +109,10 @@ help: @echo "Available targets:" @echo " all : Start servers and run tests (default, output to stdout)" @echo " ci : Run in CI mode (start servers, run tests, stop servers)" - @echo " start-servers : Start Python and Java servers in parallel (output to stdout)" + @echo " start-servers : Start Python, Java, and Go servers in parallel (output to stdout)" @echo " start-python-server: Start only the Python server" @echo " start-java-server : Start only the Java server" + @echo " start-go-server : Start only the Go server" @echo " run-tests : Run Java tests" @echo " stop-servers : Stop running servers" @echo " clean : Stop servers and clean up logs" diff --git a/test-server/go-server/.gitignore b/test-server/go-server/.gitignore new file mode 100644 index 00000000..211699c4 --- /dev/null +++ b/test-server/go-server/.gitignore @@ -0,0 +1,48 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work + +# Build output +bin/ +dist/ + +# IDE files +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS generated files +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# Logs +*.log + +# Coverage reports +coverage.out +coverage.html + +# Air (live reload) temporary files +tmp/ diff --git a/test-server/go-server/Makefile b/test-server/go-server/Makefile new file mode 100644 index 00000000..bb7eff2d --- /dev/null +++ b/test-server/go-server/Makefile @@ -0,0 +1,69 @@ +# Go Server Makefile + +.PHONY: build run clean test deps + +# Default target +all: build + +# Build the Go server +build: + go build -o bin/go-server . + +# Run the server +run: build + ./bin/go-server + +# Install dependencies +deps: + go mod tidy + go mod download + +# Clean build artifacts +clean: + rm -rf bin/ + go clean + +# Run tests (when we add them) +test: + go test ./... + +# Format code +fmt: + go fmt ./... + +# Vet code +vet: + go vet ./... + +# Run linter (requires golangci-lint) +lint: + golangci-lint run + +# Development server with auto-reload (requires air) +dev: + air + +# Check for security vulnerabilities +security: + gosec ./... + +# Generate coverage report +coverage: + go test -coverprofile=coverage.out ./... + go tool cover -html=coverage.out -o coverage.html + +# Help +help: + @echo "Available targets:" + @echo " build - Build the Go server" + @echo " run - Build and run the server" + @echo " deps - Install dependencies" + @echo " clean - Clean build artifacts" + @echo " test - Run tests" + @echo " fmt - Format code" + @echo " vet - Vet code" + @echo " lint - Run linter" + @echo " dev - Run development server with auto-reload" + @echo " security - Check for security vulnerabilities" + @echo " coverage - Generate test coverage report" + @echo " help - Show this help message" diff --git a/test-server/go-server/README.md b/test-server/go-server/README.md new file mode 100644 index 00000000..b6cfef17 --- /dev/null +++ b/test-server/go-server/README.md @@ -0,0 +1,148 @@ +# Go Server for S3 Encryption Client Test Framework + +This is a Go implementation of the S3 Encryption Client test server, part of the S3EC Generalized Robust Test Framework Machine (G-RTFM). + +## Overview + +The Go server implements the same Smithy-defined API as the Java and Python servers, providing: + +- **CreateClient**: Creates and configures S3 encryption clients +- **PutObject**: Handles encrypted object uploads to S3 +- **GetObject**: Handles encrypted object downloads from S3 + +## Architecture + +The server is built using: + +- **HTTP Framework**: Gorilla Mux for routing +- **AWS SDK**: AWS SDK for Go v2 for S3 and KMS operations +- **Concurrency**: Thread-safe client caching with sync.RWMutex +- **Error Handling**: Smithy-compliant error responses + +## API Endpoints + +### POST /client +Creates a new S3 encryption client with the provided configuration. + +**Request Body:** +```json +{ + "config": { + "enableLegacyUnauthenticatedModes": false, + "enableDelayedAuthenticationMode": false, + "enableLegacyWrappingAlgorithms": false, + "setBufferSize": 1024, + "keyMaterial": { + "rsaKey": "...", + "aesKey": "...", + "kmsKeyId": "arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012" + } + } +} +``` + +**Response:** +```json +{ + "clientId": "uuid-string" +} +``` + +### PUT /object/{bucket}/{key} +Uploads an encrypted object to S3 using the specified client. + +**Headers:** +- `ClientID`: The client ID returned from CreateClient +- `Content-Metadata`: Encryption context metadata (optional) + +**Request Body:** Raw object data + +**Response:** +```json +{ + "bucket": "bucket-name", + "key": "object-key", + "metadata": [] +} +``` + +### GET /object/{bucket}/{key} +Downloads and decrypts an object from S3 using the specified client. + +**Headers:** +- `ClientID`: The client ID returned from CreateClient +- `Content-Metadata`: Encryption context metadata (optional) + +**Response:** Raw object data with `Content-Metadata` header + +## Building and Running + +### Prerequisites + +- Go 1.21 or later +- AWS credentials configured (via AWS CLI, environment variables, or IAM roles) + +### Build + +```bash +# Install dependencies +make deps + +# Build the server +make build + +# Or build and run +make run +``` + +### Development + +```bash +# Format code +make fmt + +# Vet code +make vet + +# Run tests +make test + +# Clean build artifacts +make clean +``` + +## Configuration + +The server runs on port 8082 by default and uses the `us-west-2` AWS region. These can be modified in the source code if needed. + +## Error Handling + +The server implements Smithy-compliant error responses: + +- **GenericServerError**: For internal server errors +- **S3EncryptionClientError**: For S3 encryption client specific errors + +## Implementation Notes + +- **Client Caching**: Clients are stored in memory with UUID keys for thread-safe access +- **Metadata Handling**: Follows the same metadata string format as Java/Python servers +- **AWS Integration**: Uses AWS SDK v2 for modern Go AWS operations +- **Concurrency**: Safe for concurrent requests with proper mutex usage + +## Limitations + +- This is a basic implementation that uses standard S3 clients rather than full S3 encryption clients +- In a production implementation, you would integrate with the actual S3 encryption client library +- Memory-based client storage (not persistent across restarts) + +## Testing + +The server is designed to work with the existing test framework. It should be compatible with the Java tests that validate the Smithy API contract. + +## Future Enhancements + +- Integration with actual S3 encryption client library +- Persistent client storage +- Enhanced logging and metrics +- Configuration file support +- Health check endpoints diff --git a/test-server/go-server/go.mod b/test-server/go-server/go.mod new file mode 100644 index 00000000..014a64da --- /dev/null +++ b/test-server/go-server/go.mod @@ -0,0 +1,31 @@ +module github.com/aws/amazon-s3-encryption-client-python/test-server/go-server + +go 1.21 + +require ( + github.com/aws/amazon-s3-encryption-client-go/v3 v3.1.0 + github.com/aws/aws-sdk-go-v2 v1.24.0 + github.com/aws/aws-sdk-go-v2/config v1.26.1 + github.com/aws/aws-sdk-go-v2/service/kms v1.27.4 + github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5 + github.com/google/uuid v1.5.0 + github.com/gorilla/mux v1.8.1 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.16.12 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 // indirect + github.com/aws/smithy-go v1.19.0 // indirect +) diff --git a/test-server/go-server/go.sum b/test-server/go-server/go.sum new file mode 100644 index 00000000..4fc073e0 --- /dev/null +++ b/test-server/go-server/go.sum @@ -0,0 +1,46 @@ +github.com/aws/amazon-s3-encryption-client-go/v3 v3.1.0 h1:P4dOTmTkEb8Dj/LuAoA4bqRZZrDq4DqZQI88vdMaj18= +github.com/aws/amazon-s3-encryption-client-go/v3 v3.1.0/go.mod h1:olnwkBTbWjaJCaGOHohvJu98q40GiJZuDHLXj751mII= +github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk= +github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo= +github.com/aws/aws-sdk-go-v2/config v1.26.1 h1:z6DqMxclFGL3Zfo+4Q0rLnAZ6yVkzCRxhRMsiRQnD1o= +github.com/aws/aws-sdk-go-v2/config v1.26.1/go.mod h1:ZB+CuKHRbb5v5F0oJtGdhFTelmrxd4iWO1lf0rQwSAg= +github.com/aws/aws-sdk-go-v2/credentials v1.16.12 h1:v/WgB8NxprNvr5inKIiVVrXPuuTegM+K8nncFkr1usU= +github.com/aws/aws-sdk-go-v2/credentials v1.16.12/go.mod h1:X21k0FjEJe+/pauud82HYiQbEr9jRKY3kXEIQ4hXeTQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10/go.mod h1:K2WGI7vUvkIv1HoNbfBA1bvIZ+9kL3YVmWxeKuLQsiw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 h1:v+HbZaCGmOwnTTVS86Fleq0vPzOd7tnJGbFhP0stNLs= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9/go.mod h1:Xjqy+Nyj7VDLBtCMkQYOw1QYfAEZCVLrfI0ezve8wd4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 h1:N94sVhRACtXyVcjXxrwK1SKFIJrA9pOJ5yu2eSHnmls= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9/go.mod h1:hqamLz7g1/4EJP+GH5NBhcUMLjW+gKLQabgyz6/7WAU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 h1:ugD6qzjYtB7zM5PN/ZIeaAIyefPaD82G8+SJopgvUpw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9/go.mod h1:YD0aYBWCrPENpHolhKw2XDlTIWae2GKXT1T4o6N6hiM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 h1:/90OR2XbSYfXucBMJ4U14wrjlfleq/0SB6dZDPncgmo= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9/go.mod h1:dN/Of9/fNZet7UrQQ6kTDo/VSwKPIq94vjlU16bRARc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 h1:Nf2sHxjMJR8CSImIVCONRi4g0Su3J+TSTbS7G0pUeMU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9/go.mod h1:idky4TER38YIjr2cADF1/ugFMKvZV7p//pVeV5LZbF0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 h1:iEAeF6YC3l4FzlJPP9H3Ko1TXpdjdqWffxXjp8SY6uk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9/go.mod h1:kjsXoK23q9Z/tLBrckZLLyvjhZoS+AGrzqzUfEClvMM= +github.com/aws/aws-sdk-go-v2/service/kms v1.27.4 h1:c75pHGBV3h6WOsIjbJhLyOnlCPXzap45nbiP2Z5jk5M= +github.com/aws/aws-sdk-go-v2/service/kms v1.27.4/go.mod h1:D9FVDkZjkZnnFHymJ3fPVz0zOUlNSd0xcIIVmmrAac8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5 h1:Keso8lIOS+IzI2MkPZyK6G0LYcK3My2LQ+T5bxghEAY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5/go.mod h1:vADO6Jn+Rq4nDtfwNjhgR84qkZwiC6FqCaXdw/kYwjA= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 h1:ldSFWz9tEHAwHNmjx2Cvy1MjP5/L9kNoR0skc6wyOOM= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.5/go.mod h1:CaFfXLYL376jgbP7VKC96uFcU8Rlavak0UlAwk1Dlhc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 h1:2k9KmFawS63euAkY4/ixVNsYYwrwnd5fIvgEKkfZFNM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5/go.mod h1:W+nd4wWDVkSUIox9bacmkBP5NMFQeTJ/xqNabpzSR38= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 h1:5UYvv8JUvllZsRnfrcMQ+hJ9jNICmcgKPAO1CER25Wg= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.5/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU= +github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= +github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= diff --git a/test-server/go-server/main.go b/test-server/go-server/main.go new file mode 100644 index 00000000..23025cbd --- /dev/null +++ b/test-server/go-server/main.go @@ -0,0 +1,395 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "strings" + + "github.com/aws/amazon-s3-encryption-client-go/v3/client" + "github.com/aws/amazon-s3-encryption-client-go/v3/materials" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/google/uuid" + "github.com/gorilla/mux" +) + +// Server represents the Go test server +type Server struct { + clientCache map[string]*client.S3EncryptionClientV3 + kmsClient *kms.Client +} + +// CreateClientInput represents the input for creating a client +type CreateClientInput struct { + Config S3ECConfig `json:"config"` +} + +// CreateClientOutput represents the output for creating a client +type CreateClientOutput struct { + ClientID string `json:"clientId"` +} + +// S3ECConfig represents the S3 encryption client configuration +type S3ECConfig struct { + EnableLegacyUnauthenticatedModes bool `json:"enableLegacyUnauthenticatedModes"` + EnableDelayedAuthenticationMode bool `json:"enableDelayedAuthenticationMode"` + EnableLegacyWrappingAlgorithms bool `json:"enableLegacyWrappingAlgorithms"` + SetBufferSize int64 `json:"setBufferSize"` + KeyMaterial KeyMaterial `json:"keyMaterial"` +} + +// KeyMaterial represents the key material for encryption +type KeyMaterial struct { + RSAKey []byte `json:"rsaKey"` + AESKey []byte `json:"aesKey"` + KMSKeyID string `json:"kmsKeyId"` +} + +// PutObjectOutput represents the output for put object operation +type PutObjectOutput struct { + Bucket string `json:"bucket"` + Key string `json:"key"` + Metadata []string `json:"metadata"` +} + +// ErrorResponse represents an error response +type ErrorResponse struct { + Type string `json:"__type"` + Message string `json:"message"` +} + +// NewServer creates a new server instance +func NewServer() (*Server, error) { + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) + if err != nil { + return nil, fmt.Errorf("failed to load AWS config: %w", err) + } + + return &Server{ + clientCache: make(map[string]*client.S3EncryptionClientV3), + kmsClient: kms.NewFromConfig(cfg), + }, nil +} + +// createGenericServerError creates a generic server error response +func (s *Server) createGenericServerError(w http.ResponseWriter, message string, statusCode int) { + // Echo error to console + log.Printf("GenericServerError: %s (Status: %d)", message, statusCode) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + json.NewEncoder(w).Encode(ErrorResponse{ + Type: "software.amazon.encryption.s3#GenericServerError", + Message: message, + }) +} + +// createS3EncryptionClientError creates an S3 encryption client error response +func (s *Server) createS3EncryptionClientError(w http.ResponseWriter, message string, statusCode int) { + // Echo error to console + log.Printf("S3EncryptionClientError: %s (Status: %d)", message, statusCode) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + json.NewEncoder(w).Encode(ErrorResponse{ + Type: "software.amazon.encryption.s3#S3EncryptionClientError", + Message: message, + }) +} + +// metadataStringToMap converts metadata string to map +func metadataStringToMap(mdString string) (map[string]string, error) { + md := make(map[string]string) + if mdString == "" { + return md, nil + } + + mdList := strings.Split(mdString, ",") + for _, entry := range mdList { + // Split on "]:[" to separate key and value + parts := strings.Split(entry, "]:[") + if len(parts) == 2 { + // Remove remaining brackets from start and end + key := parts[0][1:] // Remove first character + value := parts[1][:len(parts[1])-1] // Remove last character + md[key] = value + } else { + return nil, fmt.Errorf("malformed metadata list entry: %s", entry) + } + } + return md, nil +} + +// createClient handles POST /client +func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { + log.Printf("CreateClient: Received POST /client request") + + body, err := io.ReadAll(r.Body) + if err != nil { + s.createGenericServerError(w, "Failed to read request body", http.StatusBadRequest) + return + } + + var input CreateClientInput + if err := json.Unmarshal(body, &input); err != nil { + s.createGenericServerError(w, "Invalid JSON in request body", http.StatusBadRequest) + return + } + + log.Printf("CreateClient: Parsed config - KMSKeyID: %s, EnableLegacyWrappingAlgorithms: %t", + input.Config.KeyMaterial.KMSKeyID, input.Config.EnableLegacyWrappingAlgorithms) + + // Load AWS config + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to load AWS config: %v", err), http.StatusInternalServerError) + return + } + + // Create KMS keyring based on the provided KMS key ID if available + log.Printf("CreateClient: Creating KMS keyring with key ID: %s", input.Config.KeyMaterial.KMSKeyID) + kmsClient := kms.NewFromConfig(cfg) + keyring := materials.NewKmsKeyring(kmsClient, input.Config.KeyMaterial.KMSKeyID, func(options *materials.KeyringOptions) { + options.EnableLegacyWrappingAlgorithms = input.Config.EnableLegacyWrappingAlgorithms + }) + cmm, err := materials.NewCryptographicMaterialsManager(keyring) + + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to create CMM: %v", err), http.StatusInternalServerError) + return + } + + // Create S3 encryption client + log.Printf("CreateClient: Creating S3 encryption client") + var s3EncryptionClient *client.S3EncryptionClientV3 + s3PlaintextClient := s3.NewFromConfig(cfg) + s3EncryptionClient, err = client.New(s3PlaintextClient, cmm) + + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to create S3EC: %v", err), http.StatusInternalServerError) + return + } + + // Generate client ID + clientID := uuid.New().String() + log.Printf("CreateClient: Generated client ID: %s", clientID) + + // Store client in cache + s.clientCache[clientID] = s3EncryptionClient + log.Printf("CreateClient: Stored client in cache. Total clients: %d", len(s.clientCache)) + + // Return response + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(CreateClientOutput{ + ClientID: clientID, + }) + log.Printf("CreateClient: Successfully created client %s", clientID) +} + +// putObject handles PUT /object/{bucket}/{key} +func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + bucket := vars["bucket"] + key := vars["key"] + + log.Printf("PutObject: Received PUT /object/%s/%s request", bucket, key) + + clientID := r.Header.Get("ClientID") + if clientID == "" { + s.createGenericServerError(w, "ClientID header is required", http.StatusBadRequest) + return + } + + log.Printf("PutObject: Using client ID: %s", clientID) + + // Get client from cache + client, exists := s.clientCache[clientID] + + if !exists { + s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) + return + } + + // Read body + body, err := io.ReadAll(r.Body) + if err != nil { + s.createGenericServerError(w, "Failed to read request body", http.StatusBadRequest) + return + } + + log.Printf("PutObject: Read body of size: %d bytes", len(body)) + + // Get metadata from header + metadataHeader := r.Header.Get("Content-Metadata") + encCtx, err := metadataStringToMap(metadataHeader) + + // Create context with encryption context + ctx := context.Background() + encryptionContext := context.WithValue(ctx, "EncryptionContext", encCtx) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to parse metadata: %v", err), http.StatusBadRequest) + return + } + + if len(encCtx) > 0 { + metadataJSON, _ := json.Marshal(encCtx) + log.Printf("PutObject: Using encryption context: %s", string(metadataJSON)) + } else { + log.Printf("PutObject: No encryption context provided") + } + + // Create put object input + putInput := &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: strings.NewReader(string(body)), + } + + // Add metadata if present + if len(encCtx) > 0 { + putInput.Metadata = encCtx + } + + log.Printf("PutObject: Making S3 PutObject request") + // Make the put object request using the encryption client + _, err = client.PutObject(encryptionContext, putInput) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to put object: %v", err), http.StatusInternalServerError) + return + } + + log.Printf("PutObject SUCCESS: Bucket=%s, Key=%s", bucket, key) + + // Return response + w.Header().Set("Content-Type", "application/json") + response := PutObjectOutput{ + Bucket: bucket, + Key: key, + Metadata: []string{}, // Return empty metadata list as per the model + } + json.NewEncoder(w).Encode(response) + log.Printf("PutObject: Response sent successfully") +} + +// getObject handles GET /object/{bucket}/{key} +func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + bucket := vars["bucket"] + key := vars["key"] + + log.Printf("GetObject: Received GET /object/%s/%s request", bucket, key) + + clientID := r.Header.Get("ClientID") + if clientID == "" { + s.createGenericServerError(w, "ClientID header is required", http.StatusBadRequest) + return + } + + log.Printf("GetObject: Using client ID: %s", clientID) + + // Get client from cache + client, exists := s.clientCache[clientID] + + if !exists { + s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) + return + } + + // Get metadata from header + metadataHeader := r.Header.Get("Content-Metadata") + encCtx, err := metadataStringToMap(metadataHeader) + + // Create context with encryption context + ctx := context.Background() + encryptionContext := context.WithValue(ctx, "EncryptionContext", encCtx) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to parse metadata: %v", err), http.StatusBadRequest) + return + } + + if len(encCtx) > 0 { + metadataJSON, _ := json.Marshal(encCtx) + log.Printf("GetObject: Using encryption context: %s", string(metadataJSON)) + } else { + log.Printf("GetObject: No encryption context provided") + } + + // Create get object input + getInput := &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + } + + log.Printf("GetObject: Making S3 GetObject request") + // Make the get object request using the encryption client + result, err := client.GetObject(encryptionContext, getInput) + if err != nil { + errMsg := err.Error() + // Shim the S3EC error message to the error message expected by the test server + if strings.Contains(errMsg, "to decrypt x-amz-cek-alg value `kms` you must enable legacyWrappingAlgorithms on the keyring") { + s.createS3EncryptionClientError(w, "Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms", http.StatusInternalServerError) + return + } + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to get object: %v", err), http.StatusInternalServerError) + return + } + defer result.Body.Close() + + // Read the body + body, err := io.ReadAll(result.Body) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to read object body: %v", err), http.StatusInternalServerError) + return + } + + log.Printf("GetObject: Read body of size: %d bytes", len(body)) + + // Convert metadata to string format + var metadataList []string + if result.Metadata != nil { + for k, v := range result.Metadata { + metadataList = append(metadataList, fmt.Sprintf("%s=%s", k, v)) + } + } + + metadataStr := strings.Join(metadataList, ",") + + if len(metadataList) > 0 { + log.Printf("GetObject: Retrieved metadata: %s", metadataStr) + } else { + log.Printf("GetObject: No metadata found in object") + } + + log.Printf("GetObject SUCCESS: Bucket=%s, Key=%s", bucket, key) + log.Printf("GetObject: Body content: %s", string(body)) + + // Set response headers + w.Header().Set("Content-Metadata", metadataStr) + + // Return the body as response + w.Write(body) + log.Printf("GetObject: Response sent successfully") +} + +func main() { + server, err := NewServer() + if err != nil { + log.Fatalf("Failed to create server: %v", err) + } + + r := mux.NewRouter() + + // Register routes + r.HandleFunc("/client", server.createClient).Methods("POST") + r.HandleFunc("/object/{bucket}/{key}", server.putObject).Methods("PUT") + r.HandleFunc("/object/{bucket}/{key}", server.getObject).Methods("GET") + + fmt.Println("Starting Go server on :8082...") + log.Fatal(http.ListenAndServe(":8082", r)) +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index 211269d7..325f43db 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.stream.Stream; import com.amazonaws.services.s3.model.KMSEncryptionMaterials; @@ -64,14 +65,23 @@ public class RoundTripTests { static { serverList = new ArrayList<>(2); - serverList.add(new LanguageServerTarget("Java", "8080")); - serverList.add(new LanguageServerTarget("Python", "8081")); + serverList.add(new LanguageServerTarget("Java-V3", "8080")); + serverList.add(new LanguageServerTarget("Python-V3", "8081")); + serverList.add(new LanguageServerTarget("Go-V3", "8082")); serverMap = new HashMap<>(2); - serverMap.put("Java", new LanguageServerTarget("Java", "8080")); - serverMap.put("Python", new LanguageServerTarget("Python", "8081")); + serverMap.put("Java-V3", new LanguageServerTarget("Java-V3", "8080")); + serverMap.put("Python-V3", new LanguageServerTarget("Python-V3", "8081")); + serverMap.put("Go-V3", new LanguageServerTarget("Go-V3", "8082")); } + // These languages' S3EC implementations do not validate encryption context provided to getObject. + // If the encryption context provided to getObject does not match the encryption context provided to putObject, + // these languages' implementations will not raise an error as expected. + // For now, skip tests that require this validation behavior. + private static final Set ENCRYPTION_CONTEXT_ON_DECRYPT_UNSUPPORTED = + Set.of("Go-V3"); + static public class LanguageServerTarget { public String getLanguageName() { return languageName; @@ -255,9 +265,59 @@ public void crossLanguageTestKmsWithEncCtx(LanguageServerTarget encLang, Languag @ParameterizedTest(name = "{displayName} for Encrypt: {0}, Decrypt: {1}") @MethodSource("crossLanguageClients") - public void crossLanguageTestKmsWithEncCtxMismatchFails(LanguageServerTarget encLang, LanguageServerTarget decLang) { + public void crossLanguageTestKmsWithSubsetEncCtxFails(LanguageServerTarget encLang, LanguageServerTarget decLang) { + if (ENCRYPTION_CONTEXT_ON_DECRYPT_UNSUPPORTED.contains(decLang.getLanguageName())) { + return; + } + S3ECTestServerClient encClient = testServerClientFor(encLang); + final String objectKey = "cross-lang-test-key-kms-ec-subset-fails" + encLang; + final String input = "simple-test-input"; + final Map encCtx = new HashMap<>(); + encCtx.put("user-defined-enc-ctx-key", "user-defined-enc-ctx-value"); + encCtx.put("user-defined-enc-ctx-key-2", "user-defined-enc-ctx-value-2"); + final List mdAsList = metadataMapToList(encCtx); + KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(KMS_KEY_ARN) + .build(); + CreateClientOutput encClientOutput = encClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn).build()) + .build()); + String encS3ECId = encClientOutput.getClientId(); + + encClient.putObject(PutObjectInput.builder() + .clientID(encS3ECId) + .key(objectKey) + .bucket(BUCKET) + .metadata(mdAsList) + .body(ByteBuffer.wrap(input.getBytes(StandardCharsets.UTF_8))) + .build()); + S3ECTestServerClient decClient = testServerClientFor(decLang); + CreateClientOutput decClientOutput = decClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn).build()) + .build()); + String decS3ECId = decClientOutput.getClientId(); + try { + decClient.getObject(GetObjectInput.builder() + .clientID(decS3ECId) + .bucket(BUCKET) + .key(objectKey) + .build()); + fail("Expected exception!"); + } catch (S3EncryptionClientError e) { + assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3")); + } + } + + @ParameterizedTest(name = "{displayName} for Encrypt: {0}, Decrypt: {1}") + @MethodSource("crossLanguageClients") + public void crossLanguageTestKmsWithIncorrectEncCtxFails(LanguageServerTarget encLang, LanguageServerTarget decLang) { + if (ENCRYPTION_CONTEXT_ON_DECRYPT_UNSUPPORTED.contains(decLang.getLanguageName())) { + return; + } S3ECTestServerClient encClient = testServerClientFor(encLang); - final String objectKey = "cross-lang-test-key-kms-ec-mismatch-fails" + encLang; + final String objectKey = "cross-lang-test-key-kms-ec-incorrect-fails" + encLang; final String input = "simple-test-input"; final Map encCtx = new HashMap<>(); encCtx.put("user-defined-enc-ctx-key", "user-defined-enc-ctx-value"); @@ -285,11 +345,16 @@ public void crossLanguageTestKmsWithEncCtxMismatchFails(LanguageServerTarget enc .keyMaterial(kmsKeyArn).build()) .build()); String decS3ECId = decClientOutput.getClientId(); + + final Map incorrectEncCtx = new HashMap<>(); + incorrectEncCtx.put("this-is-wrong-ec-key", "bad-value"); + var incorrectMdAsList = metadataMapToList(incorrectEncCtx); try { decClient.getObject(GetObjectInput.builder() .clientID(decS3ECId) .bucket(BUCKET) .key(objectKey) + .metadata(incorrectMdAsList) .build()); fail("Expected exception!"); } catch (S3EncryptionClientError e) { From 4d44e4f84fb998363e7ebe956e8f51c5eb52d84b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 16 Sep 2025 11:00:52 -0700 Subject: [PATCH 2/7] m --- test-server/Makefile | 72 ++++----- test-server/README.md | 11 +- test-server/go-server/.gitignore | 48 ------ test-server/go-server/Makefile | 69 -------- test-server/go-server/README.md | 148 ------------------ test-server/go-v3-server/README.md | 32 ++++ .../{go-server => go-v3-server}/go.mod | 0 .../{go-server => go-v3-server}/go.sum | 0 .../{go-server => go-v3-server}/main.go | 10 +- .../amazon/encryption/s3/RoundTripTests.java | 8 +- .../{java-server => java-v3-server}/README.md | 4 +- .../build.gradle.kts | 0 .../gradle.properties | 0 .../gradle/wrapper/gradle-wrapper.jar | Bin .../gradle/wrapper/gradle-wrapper.properties | 0 .../{java-server => java-v3-server}/gradlew | 0 .../gradlew.bat | 0 .../license.txt | 0 .../settings.gradle.kts | 0 .../smithy-build.json | 0 .../s3/CreateClientOperationImpl.java | 0 .../encryption/s3/GetObjectOperationImpl.java | 0 .../amazon/encryption/s3/MetadataUtils.java | 0 .../encryption/s3/PutObjectOperationImpl.java | 0 .../encryption/s3/S3ECJavaTestServer.java | 0 .../.gitignore | 0 .../README.md | 0 .../poetry.lock | 0 .../pyproject.toml | 0 .../src/__init__.py | 0 .../src/main.py | 0 .../tests/__init__.py | 0 32 files changed, 88 insertions(+), 314 deletions(-) delete mode 100644 test-server/go-server/.gitignore delete mode 100644 test-server/go-server/Makefile delete mode 100644 test-server/go-server/README.md create mode 100644 test-server/go-v3-server/README.md rename test-server/{go-server => go-v3-server}/go.mod (100%) rename test-server/{go-server => go-v3-server}/go.sum (100%) rename test-server/{go-server => go-v3-server}/main.go (96%) rename test-server/{java-server => java-v3-server}/README.md (70%) rename test-server/{java-server => java-v3-server}/build.gradle.kts (100%) rename test-server/{java-server => java-v3-server}/gradle.properties (100%) rename test-server/{java-server => java-v3-server}/gradle/wrapper/gradle-wrapper.jar (100%) rename test-server/{java-server => java-v3-server}/gradle/wrapper/gradle-wrapper.properties (100%) rename test-server/{java-server => java-v3-server}/gradlew (100%) rename test-server/{java-server => java-v3-server}/gradlew.bat (100%) rename test-server/{java-server => java-v3-server}/license.txt (100%) rename test-server/{java-server => java-v3-server}/settings.gradle.kts (100%) rename test-server/{java-server => java-v3-server}/smithy-build.json (100%) rename test-server/{java-server => java-v3-server}/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java (100%) rename test-server/{java-server => java-v3-server}/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java (100%) rename test-server/{java-server => java-v3-server}/src/main/java/software/amazon/encryption/s3/MetadataUtils.java (100%) rename test-server/{java-server => java-v3-server}/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java (100%) rename test-server/{java-server => java-v3-server}/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java (100%) rename test-server/{python-server => python-v3-server}/.gitignore (100%) rename test-server/{python-server => python-v3-server}/README.md (100%) rename test-server/{python-server => python-v3-server}/poetry.lock (100%) rename test-server/{python-server => python-v3-server}/pyproject.toml (100%) rename test-server/{python-server => python-v3-server}/src/__init__.py (100%) rename test-server/{python-server => python-v3-server}/src/main.py (100%) rename test-server/{python-server => python-v3-server}/tests/__init__.py (100%) diff --git a/test-server/Makefile b/test-server/Makefile index 60326393..4831d68d 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -1,6 +1,6 @@ # Makefile for S3 Encryption Client Testing -.PHONY: all start-servers start-python-server start-java-server start-go-server run-tests stop-servers clean ci check-env help +.PHONY: all start-servers start-python-v3-server start-java-v3-server start-go-v3-server run-tests stop-servers clean ci check-env help # Default target all: start-servers run-tests @@ -10,9 +10,9 @@ ci: start-servers run-tests stop-servers # Start Python server in background -start-python-server: - @echo "Starting Python server..." - cd python-server && \ +start-python-v3-server: + @echo "Starting Python V3 server..." + cd python-v3-server && \ python -m venv .venv && \ .venv/bin/python -m ensurepip && \ .venv/bin/python -m pip install -e . && \ @@ -21,36 +21,36 @@ start-python-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - .venv/bin/python src/main.py & echo $$! > ../python-server.pid + .venv/bin/python src/main.py & echo $$! > ../python-v3-server.pid @echo "Python server starting..." # Start Java server in background -start-java-server: - @echo "Starting Java server..." - cd java-server && \ +start-java-v3-server: + @echo "Starting Java V3 server..." + cd java-v3-server && \ AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./gradlew --build-cache --parallel run & echo $$! > ../java-server.pid + ./gradlew --build-cache --parallel run & echo $$! > ../java-v3-server.pid @echo "Java server starting..." # Start Go server in background -start-go-server: - @echo "Starting Go server..." - cd go-server && \ +start-go-v3-server: + @echo "Starting Go V3 server..." + cd go-v3-server && \ go mod tidy && \ AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - go run . & echo $$! > ../go-server.pid + go run . & echo $$! > ../go-v3-server.pid @echo "Go server starting..." # Start all servers in parallel start-servers: @echo "Starting servers in parallel..." - @$(MAKE) -j3 start-python-server start-java-server start-go-server + @$(MAKE) -j3 start-python-v3-server start-java-v3-server start-go-v3-server @echo "Waiting for servers to be ready..." @for i in $$(seq 1 360); do \ if nc -z localhost 8080 && nc -z localhost 8081 && nc -z localhost 8082; then \ @@ -78,46 +78,46 @@ run-tests: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./gradlew --build-cache --parallel -i integ + ./gradlew --build-cache --rerun-tasks --parallel integ @echo "Tests completed successfully" # Stop the servers stop-servers: @echo "Stopping servers..." - @if [ -f python-server.pid ]; then \ - kill $$(cat python-server.pid) 2>/dev/null || true; \ - rm python-server.pid; \ + @if [ -f python-v3-server.pid ]; then \ + kill $$(cat python-v3-server.pid) 2>/dev/null || true; \ + rm python-v3-server.pid; \ fi - @if [ -f java-server.pid ]; then \ - kill $$(cat java-server.pid) 2>/dev/null || true; \ - rm java-server.pid; \ + @if [ -f java-v3-server.pid ]; then \ + kill $$(cat java-v3-server.pid) 2>/dev/null || true; \ + rm java-v3-server.pid; \ fi - @if [ -f go-server.pid ]; then \ - kill $$(cat go-server.pid) 2>/dev/null || true; \ - rm go-server.pid; \ + @if [ -f go-v3-server.pid ]; then \ + kill $$(cat go-v3-server.pid) 2>/dev/null || true; \ + rm go-v3-server.pid; \ fi @echo "Servers stopped" # Clean up logs and pid files clean: stop-servers @echo "Cleaning up..." - @rm -f python-server.log java-server.log go-server.log + @rm -f python-v3-server.log java-v3-server.log go-v3-server.log @echo "Cleanup complete" # Help target help: @echo "Available targets:" - @echo " all : Start servers and run tests (default, output to stdout)" - @echo " ci : Run in CI mode (start servers, run tests, stop servers)" - @echo " start-servers : Start Python, Java, and Go servers in parallel (output to stdout)" - @echo " start-python-server: Start only the Python server" - @echo " start-java-server : Start only the Java server" - @echo " start-go-server : Start only the Go server" - @echo " run-tests : Run Java tests" - @echo " stop-servers : Stop running servers" - @echo " clean : Stop servers and clean up logs" - @echo " check-env : Check if required environment variables are set" - @echo " help : Show this help message" + @echo " all : Start servers and run tests (default, output to stdout)" + @echo " ci : Run in CI mode (start servers, run tests, stop servers)" + @echo " start-servers : Start all servers in parallel" + @echo " start-python-v3-server : Start only the Python V3 server" + @echo " start-java-v3-server : Start only the Java V3 server" + @echo " start-go-v3-server : Start only the Go V3 server" + @echo " run-tests : Run Java tests" + @echo " stop-servers : Stop running servers" + @echo " clean : Stop servers and clean up logs" + @echo " check-env : Check if required environment variables are set" + @echo " help : Show this help message" # Check if required environment variables are set check-env: diff --git a/test-server/README.md b/test-server/README.md index a320d1d1..4f43f1bf 100644 --- a/test-server/README.md +++ b/test-server/README.md @@ -28,11 +28,14 @@ make ci # Start Python and Java servers in parallel make start-servers -# Start only the Python server -make start-python-server +# Start only the Python S3EC V3 server +make start-python-v3-server -# Start only the Java server -make start-java-server +# Start only the Java S3EC V3 server +make start-java-v3-server + +# Start only the Go S3EC V3 server +make start-go-v3-server # Run Java tests make run-tests diff --git a/test-server/go-server/.gitignore b/test-server/go-server/.gitignore deleted file mode 100644 index 211699c4..00000000 --- a/test-server/go-server/.gitignore +++ /dev/null @@ -1,48 +0,0 @@ -# Binaries for programs and plugins -*.exe -*.exe~ -*.dll -*.so -*.dylib - -# Test binary, built with `go test -c` -*.test - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# Dependency directories (remove the comment below to include it) -# vendor/ - -# Go workspace file -go.work - -# Build output -bin/ -dist/ - -# IDE files -.vscode/ -.idea/ -*.swp -*.swo -*~ - -# OS generated files -.DS_Store -.DS_Store? -._* -.Spotlight-V100 -.Trashes -ehthumbs.db -Thumbs.db - -# Logs -*.log - -# Coverage reports -coverage.out -coverage.html - -# Air (live reload) temporary files -tmp/ diff --git a/test-server/go-server/Makefile b/test-server/go-server/Makefile deleted file mode 100644 index bb7eff2d..00000000 --- a/test-server/go-server/Makefile +++ /dev/null @@ -1,69 +0,0 @@ -# Go Server Makefile - -.PHONY: build run clean test deps - -# Default target -all: build - -# Build the Go server -build: - go build -o bin/go-server . - -# Run the server -run: build - ./bin/go-server - -# Install dependencies -deps: - go mod tidy - go mod download - -# Clean build artifacts -clean: - rm -rf bin/ - go clean - -# Run tests (when we add them) -test: - go test ./... - -# Format code -fmt: - go fmt ./... - -# Vet code -vet: - go vet ./... - -# Run linter (requires golangci-lint) -lint: - golangci-lint run - -# Development server with auto-reload (requires air) -dev: - air - -# Check for security vulnerabilities -security: - gosec ./... - -# Generate coverage report -coverage: - go test -coverprofile=coverage.out ./... - go tool cover -html=coverage.out -o coverage.html - -# Help -help: - @echo "Available targets:" - @echo " build - Build the Go server" - @echo " run - Build and run the server" - @echo " deps - Install dependencies" - @echo " clean - Clean build artifacts" - @echo " test - Run tests" - @echo " fmt - Format code" - @echo " vet - Vet code" - @echo " lint - Run linter" - @echo " dev - Run development server with auto-reload" - @echo " security - Check for security vulnerabilities" - @echo " coverage - Generate test coverage report" - @echo " help - Show this help message" diff --git a/test-server/go-server/README.md b/test-server/go-server/README.md deleted file mode 100644 index b6cfef17..00000000 --- a/test-server/go-server/README.md +++ /dev/null @@ -1,148 +0,0 @@ -# Go Server for S3 Encryption Client Test Framework - -This is a Go implementation of the S3 Encryption Client test server, part of the S3EC Generalized Robust Test Framework Machine (G-RTFM). - -## Overview - -The Go server implements the same Smithy-defined API as the Java and Python servers, providing: - -- **CreateClient**: Creates and configures S3 encryption clients -- **PutObject**: Handles encrypted object uploads to S3 -- **GetObject**: Handles encrypted object downloads from S3 - -## Architecture - -The server is built using: - -- **HTTP Framework**: Gorilla Mux for routing -- **AWS SDK**: AWS SDK for Go v2 for S3 and KMS operations -- **Concurrency**: Thread-safe client caching with sync.RWMutex -- **Error Handling**: Smithy-compliant error responses - -## API Endpoints - -### POST /client -Creates a new S3 encryption client with the provided configuration. - -**Request Body:** -```json -{ - "config": { - "enableLegacyUnauthenticatedModes": false, - "enableDelayedAuthenticationMode": false, - "enableLegacyWrappingAlgorithms": false, - "setBufferSize": 1024, - "keyMaterial": { - "rsaKey": "...", - "aesKey": "...", - "kmsKeyId": "arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012" - } - } -} -``` - -**Response:** -```json -{ - "clientId": "uuid-string" -} -``` - -### PUT /object/{bucket}/{key} -Uploads an encrypted object to S3 using the specified client. - -**Headers:** -- `ClientID`: The client ID returned from CreateClient -- `Content-Metadata`: Encryption context metadata (optional) - -**Request Body:** Raw object data - -**Response:** -```json -{ - "bucket": "bucket-name", - "key": "object-key", - "metadata": [] -} -``` - -### GET /object/{bucket}/{key} -Downloads and decrypts an object from S3 using the specified client. - -**Headers:** -- `ClientID`: The client ID returned from CreateClient -- `Content-Metadata`: Encryption context metadata (optional) - -**Response:** Raw object data with `Content-Metadata` header - -## Building and Running - -### Prerequisites - -- Go 1.21 or later -- AWS credentials configured (via AWS CLI, environment variables, or IAM roles) - -### Build - -```bash -# Install dependencies -make deps - -# Build the server -make build - -# Or build and run -make run -``` - -### Development - -```bash -# Format code -make fmt - -# Vet code -make vet - -# Run tests -make test - -# Clean build artifacts -make clean -``` - -## Configuration - -The server runs on port 8082 by default and uses the `us-west-2` AWS region. These can be modified in the source code if needed. - -## Error Handling - -The server implements Smithy-compliant error responses: - -- **GenericServerError**: For internal server errors -- **S3EncryptionClientError**: For S3 encryption client specific errors - -## Implementation Notes - -- **Client Caching**: Clients are stored in memory with UUID keys for thread-safe access -- **Metadata Handling**: Follows the same metadata string format as Java/Python servers -- **AWS Integration**: Uses AWS SDK v2 for modern Go AWS operations -- **Concurrency**: Safe for concurrent requests with proper mutex usage - -## Limitations - -- This is a basic implementation that uses standard S3 clients rather than full S3 encryption clients -- In a production implementation, you would integrate with the actual S3 encryption client library -- Memory-based client storage (not persistent across restarts) - -## Testing - -The server is designed to work with the existing test framework. It should be compatible with the Java tests that validate the Smithy API contract. - -## Future Enhancements - -- Integration with actual S3 encryption client library -- Persistent client storage -- Enhanced logging and metrics -- Configuration file support -- Health check endpoints diff --git a/test-server/go-v3-server/README.md b/test-server/go-v3-server/README.md new file mode 100644 index 00000000..2452dc93 --- /dev/null +++ b/test-server/go-v3-server/README.md @@ -0,0 +1,32 @@ +# S3EC Go V3 Test Server + +This is the Go implementation of the S3ECTestServer framework for S3EC Go V3. It provides a server implementation for testing Go S3 Encryption Client V3 functionality. + +## Overview + +The S3EC Go test server implements the S3ECTestServer service defined in the shared Smithy model. It provides endpoints for: + +- Creating S3 Encryption Clients +- Putting objects with encryption +- Getting and decrypting objects + +## Architecture + +The server is built using: + +- **HTTP Framework**: Gorilla Mux for routing +- **AWS SDK**: AWS SDK for Go v2 for S3 and KMS operations +- **Concurrency**: Thread-safe client caching with sync.RWMutex +- **Error Handling**: Smithy-compliant error responses + +## Usage + +To run the server: + +```console +gradle run +``` + +This will start the server running on port `8082`. + +The server is used as part of the testing framework to verify cross-language compatibility of the S3 Encryption Client implementations. diff --git a/test-server/go-server/go.mod b/test-server/go-v3-server/go.mod similarity index 100% rename from test-server/go-server/go.mod rename to test-server/go-v3-server/go.mod diff --git a/test-server/go-server/go.sum b/test-server/go-v3-server/go.sum similarity index 100% rename from test-server/go-server/go.sum rename to test-server/go-v3-server/go.sum diff --git a/test-server/go-server/main.go b/test-server/go-v3-server/main.go similarity index 96% rename from test-server/go-server/main.go rename to test-server/go-v3-server/main.go index 23025cbd..e79f0415 100644 --- a/test-server/go-server/main.go +++ b/test-server/go-v3-server/main.go @@ -130,6 +130,7 @@ func metadataStringToMap(mdString string) (map[string]string, error) { func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { log.Printf("CreateClient: Received POST /client request") + // Read body body, err := io.ReadAll(r.Body) if err != nil { s.createGenericServerError(w, "Failed to read request body", http.StatusBadRequest) @@ -145,14 +146,13 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { log.Printf("CreateClient: Parsed config - KMSKeyID: %s, EnableLegacyWrappingAlgorithms: %t", input.Config.KeyMaterial.KMSKeyID, input.Config.EnableLegacyWrappingAlgorithms) - // Load AWS config cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) if err != nil { s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to load AWS config: %v", err), http.StatusInternalServerError) return } - // Create KMS keyring based on the provided KMS key ID if available + // Create KMS keyring log.Printf("CreateClient: Creating KMS keyring with key ID: %s", input.Config.KeyMaterial.KMSKeyID) kmsClient := kms.NewFromConfig(cfg) keyring := materials.NewKmsKeyring(kmsClient, input.Config.KeyMaterial.KMSKeyID, func(options *materials.KeyringOptions) { @@ -306,6 +306,8 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { encCtx, err := metadataStringToMap(metadataHeader) // Create context with encryption context + // Note: S3EC Go V3 does not validate encryption context on decrypt, so the value provided here + // will not be validated against the encryption context stored on the object. ctx := context.Background() encryptionContext := context.WithValue(ctx, "EncryptionContext", encCtx) if err != nil { @@ -331,7 +333,9 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { result, err := client.GetObject(encryptionContext, getInput) if err != nil { errMsg := err.Error() - // Shim the S3EC error message to the error message expected by the test server + // Shim the S3EC error message to the error message expected by the test server. + // We don't want to change the S3EC error message but the test server expects a specific error message; + // This is the appropriate place to rewrite the error message. if strings.Contains(errMsg, "to decrypt x-amz-cek-alg value `kms` you must enable legacyWrappingAlgorithms on the keyring") { s.createS3EncryptionClientError(w, "Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms", http.StatusInternalServerError) return diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index 325f43db..3207af35 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -75,10 +75,10 @@ public class RoundTripTests { serverMap.put("Go-V3", new LanguageServerTarget("Go-V3", "8082")); } - // These languages' S3EC implementations do not validate encryption context provided to getObject. - // If the encryption context provided to getObject does not match the encryption context provided to putObject, - // these languages' implementations will not raise an error as expected. - // For now, skip tests that require this validation behavior. + // These S3EC implementations do not validate encryption context provided to getObject (i.e. on decrypt). + // If the encryption context provided to getObject does not match the encryption context on the stored object, + // these implementations will not raise an error as expected. + // For now, skip tests that expect encryption context validation on decrypt. private static final Set ENCRYPTION_CONTEXT_ON_DECRYPT_UNSUPPORTED = Set.of("Go-V3"); diff --git a/test-server/java-server/README.md b/test-server/java-v3-server/README.md similarity index 70% rename from test-server/java-server/README.md rename to test-server/java-v3-server/README.md index b2f5bb1b..e00eb496 100644 --- a/test-server/java-server/README.md +++ b/test-server/java-v3-server/README.md @@ -1,6 +1,6 @@ -# S3EC Java Test Server +# S3EC Java V3 Test Server -This is the Java implementation of the S3ECTestServer framework. It provides a server implementation for testing S3 Encryption Client functionality. +This is the Java implementation of the S3ECTestServer framework for S3EC Java V3. It provides a server implementation for testing Java S3 Encryption Client V3 functionality. ## Overview diff --git a/test-server/java-server/build.gradle.kts b/test-server/java-v3-server/build.gradle.kts similarity index 100% rename from test-server/java-server/build.gradle.kts rename to test-server/java-v3-server/build.gradle.kts diff --git a/test-server/java-server/gradle.properties b/test-server/java-v3-server/gradle.properties similarity index 100% rename from test-server/java-server/gradle.properties rename to test-server/java-v3-server/gradle.properties diff --git a/test-server/java-server/gradle/wrapper/gradle-wrapper.jar b/test-server/java-v3-server/gradle/wrapper/gradle-wrapper.jar similarity index 100% rename from test-server/java-server/gradle/wrapper/gradle-wrapper.jar rename to test-server/java-v3-server/gradle/wrapper/gradle-wrapper.jar diff --git a/test-server/java-server/gradle/wrapper/gradle-wrapper.properties b/test-server/java-v3-server/gradle/wrapper/gradle-wrapper.properties similarity index 100% rename from test-server/java-server/gradle/wrapper/gradle-wrapper.properties rename to test-server/java-v3-server/gradle/wrapper/gradle-wrapper.properties diff --git a/test-server/java-server/gradlew b/test-server/java-v3-server/gradlew similarity index 100% rename from test-server/java-server/gradlew rename to test-server/java-v3-server/gradlew diff --git a/test-server/java-server/gradlew.bat b/test-server/java-v3-server/gradlew.bat similarity index 100% rename from test-server/java-server/gradlew.bat rename to test-server/java-v3-server/gradlew.bat diff --git a/test-server/java-server/license.txt b/test-server/java-v3-server/license.txt similarity index 100% rename from test-server/java-server/license.txt rename to test-server/java-v3-server/license.txt diff --git a/test-server/java-server/settings.gradle.kts b/test-server/java-v3-server/settings.gradle.kts similarity index 100% rename from test-server/java-server/settings.gradle.kts rename to test-server/java-v3-server/settings.gradle.kts diff --git a/test-server/java-server/smithy-build.json b/test-server/java-v3-server/smithy-build.json similarity index 100% rename from test-server/java-server/smithy-build.json rename to test-server/java-v3-server/smithy-build.json diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java b/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java similarity index 100% rename from test-server/java-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java rename to test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java b/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java similarity index 100% rename from test-server/java-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java rename to test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java b/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java similarity index 100% rename from test-server/java-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java rename to test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java b/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java similarity index 100% rename from test-server/java-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java rename to test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java b/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java similarity index 100% rename from test-server/java-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java rename to test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java diff --git a/test-server/python-server/.gitignore b/test-server/python-v3-server/.gitignore similarity index 100% rename from test-server/python-server/.gitignore rename to test-server/python-v3-server/.gitignore diff --git a/test-server/python-server/README.md b/test-server/python-v3-server/README.md similarity index 100% rename from test-server/python-server/README.md rename to test-server/python-v3-server/README.md diff --git a/test-server/python-server/poetry.lock b/test-server/python-v3-server/poetry.lock similarity index 100% rename from test-server/python-server/poetry.lock rename to test-server/python-v3-server/poetry.lock diff --git a/test-server/python-server/pyproject.toml b/test-server/python-v3-server/pyproject.toml similarity index 100% rename from test-server/python-server/pyproject.toml rename to test-server/python-v3-server/pyproject.toml diff --git a/test-server/python-server/src/__init__.py b/test-server/python-v3-server/src/__init__.py similarity index 100% rename from test-server/python-server/src/__init__.py rename to test-server/python-v3-server/src/__init__.py diff --git a/test-server/python-server/src/main.py b/test-server/python-v3-server/src/main.py similarity index 100% rename from test-server/python-server/src/main.py rename to test-server/python-v3-server/src/main.py diff --git a/test-server/python-server/tests/__init__.py b/test-server/python-v3-server/tests/__init__.py similarity index 100% rename from test-server/python-server/tests/__init__.py rename to test-server/python-v3-server/tests/__init__.py From 6622859850de7ff4ca73cd4b3a427862cc2d9611 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 16 Sep 2025 11:04:12 -0700 Subject: [PATCH 3/7] m --- test-server/go-v3-server/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-server/go-v3-server/README.md b/test-server/go-v3-server/README.md index 2452dc93..6b153bfc 100644 --- a/test-server/go-v3-server/README.md +++ b/test-server/go-v3-server/README.md @@ -24,7 +24,7 @@ The server is built using: To run the server: ```console -gradle run +go run . ``` This will start the server running on port `8082`. From bcae3793c616aec5849f44a24b7d12552264a4f3 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 16 Sep 2025 11:17:48 -0700 Subject: [PATCH 4/7] Update test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java Co-authored-by: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> --- .../it/java/software/amazon/encryption/s3/RoundTripTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index 3207af35..9225b257 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -64,7 +64,7 @@ public class RoundTripTests { System.getenv("TEST_SERVER_S3_BUCKET") : "s3ec-test-server-github-bucket"; static { - serverList = new ArrayList<>(2); + serverList = new ArrayList<>(14); serverList.add(new LanguageServerTarget("Java-V3", "8080")); serverList.add(new LanguageServerTarget("Python-V3", "8081")); serverList.add(new LanguageServerTarget("Go-V3", "8082")); From bf0683c45238f3e85a97058211171f25532b0cbf Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 16 Sep 2025 11:17:57 -0700 Subject: [PATCH 5/7] Update test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java Co-authored-by: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> --- .../it/java/software/amazon/encryption/s3/RoundTripTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index 9225b257..535e4d8d 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -69,7 +69,7 @@ public class RoundTripTests { serverList.add(new LanguageServerTarget("Python-V3", "8081")); serverList.add(new LanguageServerTarget("Go-V3", "8082")); - serverMap = new HashMap<>(2); + serverMap = new HashMap<>(14); serverMap.put("Java-V3", new LanguageServerTarget("Java-V3", "8080")); serverMap.put("Python-V3", new LanguageServerTarget("Python-V3", "8081")); serverMap.put("Go-V3", new LanguageServerTarget("Go-V3", "8082")); From afecd7578fec65744ac1d2df0268371547759e4d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 16 Sep 2025 11:20:44 -0700 Subject: [PATCH 6/7] m --- test-server/go-v3-server/main.go | 59 ++++---------------------------- 1 file changed, 6 insertions(+), 53 deletions(-) diff --git a/test-server/go-v3-server/main.go b/test-server/go-v3-server/main.go index e79f0415..d201ffe2 100644 --- a/test-server/go-v3-server/main.go +++ b/test-server/go-v3-server/main.go @@ -80,7 +80,7 @@ func NewServer() (*Server, error) { // createGenericServerError creates a generic server error response func (s *Server) createGenericServerError(w http.ResponseWriter, message string, statusCode int) { // Echo error to console - log.Printf("GenericServerError: %s (Status: %d)", message, statusCode) + log.Printf("[Go V3] GenericServerError: %s (Status: %d)", message, statusCode) w.Header().Set("Content-Type", "application/json") w.WriteHeader(statusCode) @@ -93,7 +93,7 @@ func (s *Server) createGenericServerError(w http.ResponseWriter, message string, // createS3EncryptionClientError creates an S3 encryption client error response func (s *Server) createS3EncryptionClientError(w http.ResponseWriter, message string, statusCode int) { // Echo error to console - log.Printf("S3EncryptionClientError: %s (Status: %d)", message, statusCode) + log.Printf("[Go V3] S3EncryptionClientError: %s (Status: %d)", message, statusCode) w.Header().Set("Content-Type", "application/json") w.WriteHeader(statusCode) @@ -128,8 +128,6 @@ func metadataStringToMap(mdString string) (map[string]string, error) { // createClient handles POST /client func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { - log.Printf("CreateClient: Received POST /client request") - // Read body body, err := io.ReadAll(r.Body) if err != nil { @@ -143,9 +141,6 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { return } - log.Printf("CreateClient: Parsed config - KMSKeyID: %s, EnableLegacyWrappingAlgorithms: %t", - input.Config.KeyMaterial.KMSKeyID, input.Config.EnableLegacyWrappingAlgorithms) - cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) if err != nil { s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to load AWS config: %v", err), http.StatusInternalServerError) @@ -153,7 +148,6 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { } // Create KMS keyring - log.Printf("CreateClient: Creating KMS keyring with key ID: %s", input.Config.KeyMaterial.KMSKeyID) kmsClient := kms.NewFromConfig(cfg) keyring := materials.NewKmsKeyring(kmsClient, input.Config.KeyMaterial.KMSKeyID, func(options *materials.KeyringOptions) { options.EnableLegacyWrappingAlgorithms = input.Config.EnableLegacyWrappingAlgorithms @@ -166,7 +160,6 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { } // Create S3 encryption client - log.Printf("CreateClient: Creating S3 encryption client") var s3EncryptionClient *client.S3EncryptionClientV3 s3PlaintextClient := s3.NewFromConfig(cfg) s3EncryptionClient, err = client.New(s3PlaintextClient, cmm) @@ -178,18 +171,15 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { // Generate client ID clientID := uuid.New().String() - log.Printf("CreateClient: Generated client ID: %s", clientID) // Store client in cache s.clientCache[clientID] = s3EncryptionClient - log.Printf("CreateClient: Stored client in cache. Total clients: %d", len(s.clientCache)) // Return response w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(CreateClientOutput{ ClientID: clientID, }) - log.Printf("CreateClient: Successfully created client %s", clientID) } // putObject handles PUT /object/{bucket}/{key} @@ -198,16 +188,12 @@ func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { bucket := vars["bucket"] key := vars["key"] - log.Printf("PutObject: Received PUT /object/%s/%s request", bucket, key) - clientID := r.Header.Get("ClientID") if clientID == "" { s.createGenericServerError(w, "ClientID header is required", http.StatusBadRequest) return } - log.Printf("PutObject: Using client ID: %s", clientID) - // Get client from cache client, exists := s.clientCache[clientID] @@ -223,8 +209,6 @@ func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { return } - log.Printf("PutObject: Read body of size: %d bytes", len(body)) - // Get metadata from header metadataHeader := r.Header.Get("Content-Metadata") encCtx, err := metadataStringToMap(metadataHeader) @@ -237,13 +221,6 @@ func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { return } - if len(encCtx) > 0 { - metadataJSON, _ := json.Marshal(encCtx) - log.Printf("PutObject: Using encryption context: %s", string(metadataJSON)) - } else { - log.Printf("PutObject: No encryption context provided") - } - // Create put object input putInput := &s3.PutObjectInput{ Bucket: aws.String(bucket), @@ -256,7 +233,6 @@ func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { putInput.Metadata = encCtx } - log.Printf("PutObject: Making S3 PutObject request") // Make the put object request using the encryption client _, err = client.PutObject(encryptionContext, putInput) if err != nil { @@ -264,7 +240,7 @@ func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { return } - log.Printf("PutObject SUCCESS: Bucket=%s, Key=%s", bucket, key) + log.Printf("[Go V3] PutObject SUCCESS: Bucket=%s, Key=%s", bucket, key) // Return response w.Header().Set("Content-Type", "application/json") @@ -274,7 +250,6 @@ func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { Metadata: []string{}, // Return empty metadata list as per the model } json.NewEncoder(w).Encode(response) - log.Printf("PutObject: Response sent successfully") } // getObject handles GET /object/{bucket}/{key} @@ -283,16 +258,12 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { bucket := vars["bucket"] key := vars["key"] - log.Printf("GetObject: Received GET /object/%s/%s request", bucket, key) - clientID := r.Header.Get("ClientID") if clientID == "" { s.createGenericServerError(w, "ClientID header is required", http.StatusBadRequest) return } - log.Printf("GetObject: Using client ID: %s", clientID) - // Get client from cache client, exists := s.clientCache[clientID] @@ -315,20 +286,12 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { return } - if len(encCtx) > 0 { - metadataJSON, _ := json.Marshal(encCtx) - log.Printf("GetObject: Using encryption context: %s", string(metadataJSON)) - } else { - log.Printf("GetObject: No encryption context provided") - } - // Create get object input getInput := &s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), } - log.Printf("GetObject: Making S3 GetObject request") // Make the get object request using the encryption client result, err := client.GetObject(encryptionContext, getInput) if err != nil { @@ -352,8 +315,6 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { return } - log.Printf("GetObject: Read body of size: %d bytes", len(body)) - // Convert metadata to string format var metadataList []string if result.Metadata != nil { @@ -364,27 +325,19 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { metadataStr := strings.Join(metadataList, ",") - if len(metadataList) > 0 { - log.Printf("GetObject: Retrieved metadata: %s", metadataStr) - } else { - log.Printf("GetObject: No metadata found in object") - } - - log.Printf("GetObject SUCCESS: Bucket=%s, Key=%s", bucket, key) - log.Printf("GetObject: Body content: %s", string(body)) + log.Printf("[Go V3] GetObject SUCCESS: Bucket=%s, Key=%s", bucket, key) // Set response headers w.Header().Set("Content-Metadata", metadataStr) // Return the body as response w.Write(body) - log.Printf("GetObject: Response sent successfully") } func main() { server, err := NewServer() if err != nil { - log.Fatalf("Failed to create server: %v", err) + log.Fatalf("[Go V3] Failed to create Go V3 server: %v", err) } r := mux.NewRouter() @@ -394,6 +347,6 @@ func main() { r.HandleFunc("/object/{bucket}/{key}", server.putObject).Methods("PUT") r.HandleFunc("/object/{bucket}/{key}", server.getObject).Methods("GET") - fmt.Println("Starting Go server on :8082...") + fmt.Println("[Go V3] Starting Go V3 server on :8082...") log.Fatal(http.ListenAndServe(":8082", r)) } From 2e594b615ee40028d2c13b9a5ace67bb410f83d4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 17 Sep 2025 09:40:04 -0700 Subject: [PATCH 7/7] m --- test-server/go-v3-server/README.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/test-server/go-v3-server/README.md b/test-server/go-v3-server/README.md index 6b153bfc..cf1692b6 100644 --- a/test-server/go-v3-server/README.md +++ b/test-server/go-v3-server/README.md @@ -10,15 +10,6 @@ The S3EC Go test server implements the S3ECTestServer service defined in the sha - Putting objects with encryption - Getting and decrypting objects -## Architecture - -The server is built using: - -- **HTTP Framework**: Gorilla Mux for routing -- **AWS SDK**: AWS SDK for Go v2 for S3 and KMS operations -- **Concurrency**: Thread-safe client caching with sync.RWMutex -- **Error Handling**: Smithy-compliant error responses - ## Usage To run the server: