diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a07c9a96..f99b5e2a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,8 +5,8 @@ on: # Optional inputs that can be provided when calling this workflow inputs: python-version: - description: 'Python version to use' - default: '3.11' + description: "Python version to use" + default: "3.11" required: false type: string @@ -16,7 +16,7 @@ jobs: permissions: id-token: write contents: read - + steps: - name: Checkout code uses: actions/checkout@v5 @@ -32,7 +32,7 @@ jobs: repository: awslabs/aws-sdk-cpp-staging ref: fire-egg-dev path: test-server/cpp-v2-transition-server/aws-sdk-cpp/ - + - name: Set up Python uses: actions/setup-python@v5 with: @@ -41,23 +41,28 @@ jobs: - name: Set up Ruby uses: ruby/setup-ruby@v1 with: - ruby-version: '3.4' - + ruby-version: "3.4" + - name: Set up PHP with Composer uses: shivammathur/setup-php@verbose with: - php-version: '8.1' - + php-version: "8.1" + - name: Install PHP V2 dependencies working-directory: ./test-server/php-v2-server shell: bash run: composer install + - name: Install PHP V2 Transition dependencies + working-directory: ./test-server/php-v2-transition-server + shell: bash + run: composer install + - name: Install PHP V3 dependencies working-directory: ./test-server/php-v3-server shell: bash run: composer install - + - name: Install Go uses: actions/setup-go@v5 with: @@ -71,10 +76,10 @@ jobs: key: ${{ runner.os }}-uv-${{ hashFiles('**/pyproject.toml') }} restore-keys: | ${{ runner.os }}-uv- - + - name: Install Uv run: pip install uv - + # Cache Gradle dependencies and build outputs - name: Cache Gradle packages uses: actions/cache@v4 @@ -87,25 +92,25 @@ jobs: key: ${{ runner.os }}-gradle-${{ hashFiles('test-server/java-v3-server/**/*.gradle*', 'test-server/java-tests/**/gradle-wrapper.properties', 'test-server/java-tests/**/*.gradle*', 'test-server/java-v3-server/**/gradle-wrapper.properties') }} restore-keys: | ${{ runner.os }}-gradle- - + - name: Install dependencies run: make install - + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role aws-region: us-west-2 - + - name: Run unit tests run: make test-unit - + - name: Run integration tests run: make test-integration env: CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} - + - name: Run test-server tests run: cd test-server && make ci env: diff --git a/.gitmodules b/.gitmodules index 68816e01..dc1094d8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,7 +7,7 @@ [submodule "test-server/php-v2-server/local-php-sdk"] path = test-server/php-v2-server/local-php-sdk url = git@github.com:aws/private-aws-sdk-php-staging.git - branch = s3ec/transitional + branch = master [submodule "test-server/php-v3-server/local-php-sdk"] path = test-server/php-v3-server/local-php-sdk url = git@github.com:aws/private-aws-sdk-php-staging.git @@ -24,3 +24,7 @@ path = test-server/specification url = git@github.com:awslabs/private-aws-encryption-sdk-specification-staging.git branch = fire-egg-staging +[submodule "test-server/php-v2-transition-server/local-php-sdk"] + path = test-server/php-v2-transition-server/local-php-sdk + url = git@github.com:aws/private-aws-sdk-php-staging.git + branch = s3ec/transitional diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index 62371b56..4818b2a8 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -75,7 +75,7 @@ public class TestUtils { // Sets of unsupported features by language public static final Set ENCRYPTION_CONTEXT_ON_DECRYPT_UNSUPPORTED = - Set.of(GO_V3_CURRENT, PHP_V2_CURRENT, PHP_V3, NET_V2_CURRENT, NET_V3); + Set.of(GO_V3_CURRENT, PHP_V2_CURRENT, PHP_V2_TRANSITION, PHP_V3, NET_V2_CURRENT, NET_V3); public static final Set ENCRYPTION_CONTEXT_ON_ENCRYPT_UNSUPPORTED = Set.of(NET_V2_CURRENT, NET_V3); @@ -131,7 +131,7 @@ public class TestUtils { // servers.put(NET_V2_TRANSITION, new LanguageServerTarget(NET_V2_TRANSITION, "8096")); servers.put(CPP_V2_TRANSITION, new LanguageServerTarget(CPP_V2_TRANSITION, "8097")); // servers.put(RUBY_V2_TRANSITION, new LanguageServerTarget(RUBY_V2_TRANSITION, "8098")); - // servers.put(PHP_V2_TRANSITION, new LanguageServerTarget(PHP_V2_TRANSITION, "8099")); + servers.put(PHP_V2_TRANSITION, new LanguageServerTarget(PHP_V2_TRANSITION, "8099")); servers.put(JAVA_V4, new LanguageServerTarget(JAVA_V4, "8090")); serverMap = filterServers(servers); } diff --git a/test-server/php-v2-transition-server/.duvet/.gitignore b/test-server/php-v2-transition-server/.duvet/.gitignore new file mode 100644 index 00000000..93956e36 --- /dev/null +++ b/test-server/php-v2-transition-server/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ \ No newline at end of file diff --git a/test-server/php-v2-transition-server/.duvet/config.toml b/test-server/php-v2-transition-server/.duvet/config.toml new file mode 100644 index 00000000..64b00927 --- /dev/null +++ b/test-server/php-v2-transition-server/.duvet/config.toml @@ -0,0 +1,24 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "local-php-sdk/src/S3/**/*.php" + +[[source]] +pattern = "local-php-sdk/src/Crypto/**/*.php" + +# Include required specifications here +[[specification]] +source = "../specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "../specification/s3-encryption/data-format/metadata-strategy.md" +[[specification]] +source = "../specification/s3-encryption/encryption.md" +[[specification]] +source = "../specification/s3-encryption/key-derivation.md" + +[report.html] +enabled = true + +# Enable snapshots to prevent requirement coverage regressions +[report.snapshot] +enabled = false diff --git a/test-server/php-v2-transition-server/.gitignore b/test-server/php-v2-transition-server/.gitignore new file mode 100644 index 00000000..07108589 --- /dev/null +++ b/test-server/php-v2-transition-server/.gitignore @@ -0,0 +1,4 @@ +vendor/* +cookies.txt +server.pid +composer.lock \ No newline at end of file diff --git a/test-server/php-v2-transition-server/Makefile b/test-server/php-v2-transition-server/Makefile new file mode 100644 index 00000000..536d5cdb --- /dev/null +++ b/test-server/php-v2-transition-server/Makefile @@ -0,0 +1,30 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: start-server stop-server wait-for-server + +PID_FILE := server.pid +PORT := 8099 + +start-server: + @echo "Starting PHP V2 server..." + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + composer run start & echo $$! > $(PID_FILE) + @echo "PHP V2 server starting..." + +stop-server: + @if [ -f $(PID_FILE) ]; then \ + kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm $(PID_FILE); \ + fi + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT) + +duvet: + duvet report + +view-report-mac: + open .duvet/reports/report.html diff --git a/test-server/php-v2-transition-server/composer.json b/test-server/php-v2-transition-server/composer.json new file mode 100644 index 00000000..6a0f263b --- /dev/null +++ b/test-server/php-v2-transition-server/composer.json @@ -0,0 +1,36 @@ +{ + "name": "aws/s3ec-php-v2-transition-test-server", + "description": "PHP V2 Transition implementation of the S3EC Test Server framework", + "type": "project", + "license": "Apache-2.0", + "repositories": [ + { + "type": "path", + "url": "./local-php-sdk", + "options": { + "symlink": true + } + } + ], + "require": { + "php": ">=7.4", + "aws/aws-sdk-php": "@dev", + "ramsey/uuid": "^4.9" + }, + "autoload": { + "psr-4": { + "S3EC\\PhpV2Server\\": "src/" + } + }, + "scripts": { + "start": [ + "php -S 0.0.0.0:8099 src/index.php" + ] + }, + "config": { + "optimize-autoloader": true, + "platform": { + "php": "8.1" + } + } +} \ No newline at end of file diff --git a/test-server/php-v2-transition-server/local-php-sdk b/test-server/php-v2-transition-server/local-php-sdk new file mode 160000 index 00000000..d78bd3b2 --- /dev/null +++ b/test-server/php-v2-transition-server/local-php-sdk @@ -0,0 +1 @@ +Subproject commit d78bd3b221890aac679ec3b6cb5abcb01fd42699 diff --git a/test-server/php-v2-transition-server/src/client.php b/test-server/php-v2-transition-server/src/client.php new file mode 100644 index 00000000..44fe1b39 --- /dev/null +++ b/test-server/php-v2-transition-server/src/client.php @@ -0,0 +1,68 @@ +toString(); + $kmsKeyId = $keyMaterial["kmsKeyId"] ?? null; + + if ($configData == []) { + return GenericServerError("Invalid config in request body", 400); + } + if (($keyMaterial || $kmsKeyId) === null) { + return GenericServerError("Invalid keyMaterial in config", 400); + } + + // Store client configuration instead of objects (AWS objects can't be serialized) + $_SESSION['s3ecCache'][$clientId] = [ + 's3Config' => [ + 'region' => 'us-west-2', + 'version' => 'latest', + 'http' => [ + 'debug' => false, + 'verify' => true, + 'curl' => [ + CURLOPT_VERBOSE => false, + CURLOPT_NOPROGRESS => true + ] + ] + ], + 'kmsConfig' => [ + 'region' => 'us-west-2', + 'version' => 'latest', + 'http' => [ + 'debug' => false, + 'verify' => true, + 'curl' => [ + CURLOPT_VERBOSE => false, + CURLOPT_NOPROGRESS => true + ] + ] + ], + 'kmsKeyId' => $kmsKeyId, + 'legacy' => $legacyAlgorithms, + 'created' => time() + ]; + + // Auto-update cookies.txt with current session ID so tests can access cached clients + writeSessionIdToCookiesFile(session_id()); + + header("Content-Type: application/json"); + return json_encode([ + 'clientId' => $clientId, + ]); +} diff --git a/test-server/php-v2-transition-server/src/errors.php b/test-server/php-v2-transition-server/src/errors.php new file mode 100644 index 00000000..67449c11 --- /dev/null +++ b/test-server/php-v2-transition-server/src/errors.php @@ -0,0 +1,42 @@ + 'GenericServerError', + 'message' => $message + ]; + + return json_encode($errorResponse); +} + +/** + * Used for modeled errors, e.g. errors thrown by the S3EC + * Tests SHOULD expect this error in negative tests. + * + * @param string $message The error message to include in the response + * @return string JSON-encoded error response + */ +function S3EncryptionClientError($message) +{ + http_response_code(500); + header('Content-Type: application/json'); + + $errorResponse = [ + "__type" => "software.amazon.encryption.s3#S3EncryptionClientError", + 'message' => $message + ]; + + return json_encode($errorResponse); +} diff --git a/test-server/php-v2-transition-server/src/get_object.php b/test-server/php-v2-transition-server/src/get_object.php new file mode 100644 index 00000000..41875f54 --- /dev/null +++ b/test-server/php-v2-transition-server/src/get_object.php @@ -0,0 +1,85 @@ +getObject([ + '@SecurityProfile' => $legacy, + '@MaterialsProvider' => $materialProvider, + '@KmsEncryptionContext' => $encryptionContext, + 'Bucket' => $bucket, + 'Key' => $key, + ]); + + // Capture and discard any unwanted output from AWS SDK + $unwantedOutput = ob_get_clean(); + if (!empty($unwantedOutput)) { + error_log("AWS SDK produced unexpected output: " . strlen($unwantedOutput) . " bytes"); + } + + $body = $result['Body']->getContents(); + $formattedMetadata = formatMetadataForResponse($result["Metadata"]); + + // Now set headers safely + header("Content-Metadata: " . $formattedMetadata); + header("Content-Type: application/octet-stream"); + header("Content-Length: " . strlen($body)); + return $body; + } catch (InvalidArgumentException $e) { + // Clean up output buffer if still active + if (ob_get_level()) { + ob_end_clean(); + } + return GenericServerError("Invalid argument: " . $e->getMessage(), 400); + } catch (Exception $e) { + // Clean up output buffer if still active + if (ob_get_level()) { + ob_end_clean(); + } + if (strpos($e->getMessage(), "@SecurityProfile=V2") !== false) { + return S3EncryptionClientError($e->getMessage() . " " . "Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms"); + } else { + return GenericServerError("Server error: " . $e->getMessage(), 500); + } + } +} diff --git a/test-server/php-v2-transition-server/src/index.php b/test-server/php-v2-transition-server/src/index.php new file mode 100644 index 00000000..cc5dee29 --- /dev/null +++ b/test-server/php-v2-transition-server/src/index.php @@ -0,0 +1,294 @@ += 7 && $parts[5] === 'PHPSESSID') { + error_log("Found session ID in cookies.txt: " . $parts[6]); + return $parts[6]; // Return the session ID value + } + } + + error_log("No PHPSESSID found in cookies.txt file"); + return null; +} + +// Function to write session ID to cookies.txt file +function writeSessionIdToCookiesFile($sessionId) +{ + $cookiesFile = __DIR__ . '/../cookies.txt'; + + // Create Netscape cookie format entry + $cookieLine = "localhost\tFALSE\t/\tFALSE\t0\tPHPSESSID\t$sessionId"; + + // Write header and cookie entry + $content = "# Netscape HTTP Cookie File\n"; + $content .= "# https://curl.se/docs/http-cookies.html\n"; + $content .= "# This file was generated by libcurl! Edit at your own risk.\n\n"; + $content .= $cookieLine . "\n"; + + $result = file_put_contents($cookiesFile, $content); + + if ($result === false) { + error_log("Failed to write session ID to cookies.txt file: $cookiesFile"); + return false; + } + + error_log("Successfully wrote session ID to cookies.txt: $sessionId"); + return true; +} + +set_time_limit(600); +// Start session to persist cache across requests +// First try to use session ID from cookies.txt if available +$sessionId = getSessionIdFromCookiesFile(); +if ($sessionId) { + session_id($sessionId); +} +session_start(); + +// Initialize session cache if it doesn't exist +if (!isset($_SESSION['s3ecCache'])) { + $_SESSION['s3ecCache'] = []; +} + +// Simple router class +class SimpleRouter +{ + private $routes = []; + + public function addRoute($method, $path, $handler) + { + $this->routes[] = [ + 'method' => strtoupper($method), + 'path' => $path, + 'handler' => $handler + ]; + } + + public function handleRequest() + { + $method = $_SERVER['REQUEST_METHOD']; + $path = parse_url($_SERVER['REQUEST_URI'], PHP_URL_PATH); + + foreach ($this->routes as $route) { + if ($route['method'] === $method) { + $params = $this->matchPathWithParams($route['path'], $path); + if ($params !== false) { + return call_user_func($route['handler'], $params); + } + } + } + + // Default 404 response + http_response_code(404); + return json_encode(['error' => 'Not Found']); + } + + private function matchPathWithParams($routePath, $requestPath) + { + // Handle exact matches first (for routes without parameters) + if ($routePath === $requestPath) { + return []; + } + + // Convert route path like '/object/{bucket}/{key}' to regex + $pattern = preg_replace('/\{([^}]+)\}/', '([^/]+)', $routePath); + $pattern = '/^' . str_replace('/', '\/', $pattern) . '$/'; + + if (preg_match($pattern, $requestPath, $matches)) { + array_shift($matches); // Remove full match + + // Extract parameter names + preg_match_all('/\{([^}]+)\}/', $routePath, $paramNames); + $params = []; + + foreach ($paramNames[1] as $index => $paramName) { + $params[$paramName] = $matches[$index] ?? null; + } + + return $params; + } + + return false; + } +} + +// Helper function to get cached client by ID +function getCachedClient($clientId) +{ + if (!isset($_SESSION['s3ecCache'][$clientId])) { + return null; + } + + $config = $_SESSION['s3ecCache'][$clientId]; + + // Recreate the AWS clients from stored configuration + $s3Client = new S3Client($config['s3Config']); + $encryptionClient = new S3EncryptionClientV2($s3Client); + + $kmsClient = new KmsClient($config['kmsConfig']); + $materialsProvider = new KmsMaterialsProviderV2($kmsClient, $config['kmsKeyId']); + + return [ + 'encryptionClient' => $encryptionClient, + 'materialsProvider' => $materialsProvider, + 'config' => $config + ]; +} + +function createDefaultClientTuple(): array +{ + $s3Client = new S3Client([ + 'region' => 'us-west-2', + 'version' => 'latest', + 'http' => [ + 'debug' => false, + 'verify' => true, + 'curl' => [ + CURLOPT_VERBOSE => false, + CURLOPT_NOPROGRESS => true + ] + ] + ]); + $encryptionClient = new S3EncryptionClientV2($s3Client); + + $kmsClient = new KmsClient([ + 'region' => 'us-west-2', + 'version' => 'latest', + 'http' => [ + 'debug' => false, + 'verify' => true, + 'curl' => [ + CURLOPT_VERBOSE => false, + CURLOPT_NOPROGRESS => true + ] + ] + ]); + $materialsProvider = new KmsMaterialsProviderV2($kmsClient, 'arn:aws:kms:us-west-2:370957321024:alias/S3EC-Test-Server-Github-KMS-Key'); + + return [ + 'encryptionClient' => $encryptionClient, + 'materialsProvider' => $materialsProvider + ]; +} + +function metadataStringToMap($metadata): array +{ + $md = []; + + if (empty($metadata)) { + return $md; + } + + $mdList = explode(',', $metadata); + + foreach ($mdList as $entry) { + $parts = explode(']:[', $entry); + + if (count($parts) === 2) { + $key = substr($parts[0], 1); + $value = substr($parts[1], 0, -1); + $md[$key] = $value; + } else { + throw new InvalidArgumentException("Malformed metadata list entry: " . $entry); + } + } + + return $md; +} +function formatMetadataForResponse($metadata) +{ + $metadataList = []; + // Handle different metadata input types + if (is_array($metadata)) { + // If it's an associative array (like Python dict) + foreach ($metadata as $key => $value) { + $metadataList[] = $key . '=' . $value; + } + } elseif (is_string($metadata) && !empty($metadata)) { + // If it's already a string, assume it's in the correct format + return $metadata; + } + + // Convert array to comma-separated string + return implode(',', $metadataList); +} + +// Initialize router +$router = new SimpleRouter(); + +// Add basic routes +$router->addRoute('GET', '/', function () { + return json_encode([ + 'service' => 'S3EC PHP v2 Test Server', + 'status' => 'running', + 'port' => 8087, + 'endpoints' => [ + 'GET /' => 'Server status', + 'POST /client' => 'Create an S3EncryptionClient and cache it.', + 'GET /object/{bucket}/{key}' => 'Handle GET requests to /object/{bucket}/{key} by using the S3EncryptionClient to make a GetObject request to S3.', + 'PUT /object/{bucket}/{key}' => 'Handle PUT requests to /object/{bucket}/{key} by using the S3EncryptionClient to make a PutObject request to S3.', + ] + ]); +}); + +$router->addRoute('GET', '/cache', function () { + return json_encode([ + 'sessionId' => session_id(), + 'sessionStatus' => session_status(), + 'totalCachedClients' => count($_SESSION['s3ecCache'] ?? []), + 'allClientIds' => array_keys($_SESSION['s3ecCache'] ?? []), + 'cacheDetails' => $_SESSION['s3ecCache'] ?? [] + ]); +}); + +$router->addRoute('GET', '/object/{bucket}/{key}', function ($params) { + return handleGetObject($params); +}); + +$router->addRoute('PUT', '/object/{bucket}/{key}', function ($params) { + return handlePutObject($params); +}); + +$router->addRoute('POST', '/client', function () { + return handleCreateClient(); +}); + +// Handle the request and output response +$result = $router->handleRequest(); +if ($result !== false) { + echo $result; +} diff --git a/test-server/php-v2-transition-server/src/put_object.php b/test-server/php-v2-transition-server/src/put_object.php new file mode 100644 index 00000000..c7de4bb4 --- /dev/null +++ b/test-server/php-v2-transition-server/src/put_object.php @@ -0,0 +1,72 @@ + 'gcm', + 'KeySize' => 256, + ]; + $legacyConfig = $s3ecClientTuple["legacy"] ?? false; + $legacy = null; + if ($legacyConfig === false) { + $legacy = "V2"; + } else { + $legacy = "V2_AND_LEGACY"; + } + + try { + $result = $s3ec->putObject([ + '@SecurityProfile' => $legacy, + '@MaterialsProvider' => $materialProvider, + '@KmsEncryptionContext' => $encryptionContext, + '@CipherOptions' => $cipherOptions, + 'Bucket' => $bucket, + 'Key' => $key, + 'Body' => $rawBody, + ]); + + header("Content-Type: application/json"); + return json_encode([ + "bucket" => $bucket, + "key" => $key, + // php for some reason blows java's heap if we pass the metadata + // "metadata" => $encryptionContext + ]); + + } catch (InvalidArgumentException $e) { + return S3EncryptionClientError("Invalid argument: " . $e->getMessage()); + } catch (Exception $e) { + return GenericServerError("Server error: " . $e->getMessage()); + } +}