From 374c3bcf32e2bb9863671d657750c30a93bce381 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 6 Aug 2025 13:32:14 -0700 Subject: [PATCH 01/81] chore: add CDK for basic CI resources --- cdk/.gitignore | 8 + cdk/.npmignore | 6 + cdk/README.md | 14 + cdk/cdk.json | 57 + cdk/jest.config.js | 8 + cdk/lib/cdk-stack.ts | 142 ++ cdk/package-lock.json | 4382 +++++++++++++++++++++++++++++++++++++++++ cdk/package.json | 27 + cdk/test/cdk.test.ts | 17 + cdk/tsconfig.json | 31 + 10 files changed, 4692 insertions(+) create mode 100644 cdk/.gitignore create mode 100644 cdk/.npmignore create mode 100644 cdk/README.md create mode 100644 cdk/cdk.json create mode 100644 cdk/jest.config.js create mode 100644 cdk/lib/cdk-stack.ts create mode 100644 cdk/package-lock.json create mode 100644 cdk/package.json create mode 100644 cdk/test/cdk.test.ts create mode 100644 cdk/tsconfig.json diff --git a/cdk/.gitignore b/cdk/.gitignore new file mode 100644 index 00000000..f60797b6 --- /dev/null +++ b/cdk/.gitignore @@ -0,0 +1,8 @@ +*.js +!jest.config.js +*.d.ts +node_modules + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/cdk/.npmignore b/cdk/.npmignore new file mode 100644 index 00000000..c1d6d45d --- /dev/null +++ b/cdk/.npmignore @@ -0,0 +1,6 @@ +*.ts +!*.d.ts + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/cdk/README.md b/cdk/README.md new file mode 100644 index 00000000..320efc02 --- /dev/null +++ b/cdk/README.md @@ -0,0 +1,14 @@ +# Welcome to your CDK TypeScript project + +This is a blank project for CDK development with TypeScript. + +The `cdk.json` file tells the CDK Toolkit how to execute your app. + +## Useful commands + +* `npm run build` compile typescript to js +* `npm run watch` watch for changes and compile +* `npm run test` perform the jest unit tests +* `cdk deploy` deploy this stack to your default AWS account/region +* `cdk diff` compare deployed stack with current state +* `cdk synth` emits the synthesized CloudFormation template diff --git a/cdk/cdk.json b/cdk/cdk.json new file mode 100644 index 00000000..a7260af7 --- /dev/null +++ b/cdk/cdk.json @@ -0,0 +1,57 @@ +{ + "app": "npx ts-node --prefer-ts-exts bin/cdk.ts", + "watch": { + "include": [ + "**" + ], + "exclude": [ + "README.md", + "cdk*.json", + "**/*.d.ts", + "**/*.js", + "tsconfig.json", + "package*.json", + "yarn.lock", + "node_modules", + "test" + ] + }, + "context": { + "@aws-cdk/aws-lambda:recognizeLayerVersion": true, + "@aws-cdk/core:checkSecretUsage": true, + "@aws-cdk/core:target-partitions": [ + "aws", + "aws-cn" + ], + "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, + "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, + "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, + "@aws-cdk/aws-iam:minimizePolicies": true, + "@aws-cdk/core:validateSnapshotRemovalPolicy": true, + "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, + "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, + "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, + "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, + "@aws-cdk/core:enablePartitionLiterals": true, + "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, + "@aws-cdk/aws-iam:standardizedServicePrincipals": true, + "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, + "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, + "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, + "@aws-cdk/aws-route53-patters:useCertificate": true, + "@aws-cdk/customresources:installLatestAwsSdkDefault": false, + "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, + "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, + "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, + "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, + "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, + "@aws-cdk/aws-redshift:columnId": true, + "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, + "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, + "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, + "@aws-cdk/aws-kms:aliasNameRef": true, + "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, + "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, + "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true + } +} diff --git a/cdk/jest.config.js b/cdk/jest.config.js new file mode 100644 index 00000000..08263b89 --- /dev/null +++ b/cdk/jest.config.js @@ -0,0 +1,8 @@ +module.exports = { + testEnvironment: 'node', + roots: ['/test'], + testMatch: ['**/*.test.ts'], + transform: { + '^.+\\.tsx?$': 'ts-jest' + } +}; diff --git a/cdk/lib/cdk-stack.ts b/cdk/lib/cdk-stack.ts new file mode 100644 index 00000000..88f31473 --- /dev/null +++ b/cdk/lib/cdk-stack.ts @@ -0,0 +1,142 @@ +import * as cdk from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import { + Alias, + Key +} from "aws-cdk-lib/aws-kms"; +import { + Effect, + Role, + PolicyDocument, + PolicyStatement, + FederatedPrincipal, + ManagedPolicy, +} from "aws-cdk-lib/aws-iam"; +import { + BlockPublicAccess, + BlockPublicAccessOptions, + Bucket, +} from 'aws-cdk-lib/aws-s3'; + +export class S3ECPythonGithub extends cdk.Stack { + constructor(scope: Construct, id: string, props?: cdk.StackProps) { + super(scope, id, props); + + // KMS Key - default policy is fine, + // we use IAM to manage key permissions + const S3ECGithubKMSKey = new Key( + this, + "S3ECGithubKMSKey", + { + enableKeyRotation: true, + description: "KMS Key for GitHub Action Workflow", + } + ) + + // KMS alias + const S3ECGithubKMSKeyAlias = new Alias( + this, + "S3ECGithubKMSKeyAlias", + { + aliasName: "alias/S3EC-Python-Github-KMS-Key", + targetKey: S3ECGithubKMSKey + } + ) + + // S3 bucket + const AccessConfiguration: BlockPublicAccessOptions = { + blockPublicAcls: false, + blockPublicPolicy: false, + ignorePublicAcls: false, + restrictPublicBuckets: false + } + const S3ECGithubTestS3Bucket = new Bucket( + this, + "S3ECGithubTestS3Bucket", + { + bucketName: "s3ec-python-github-test-bucket", + blockPublicAccess: new BlockPublicAccess(AccessConfiguration) + } + ) + + // S3 bucket policy + const S3ECGithubS3BucketPolicy = new ManagedPolicy( + this, + "S3EC-Python-Github-S3-Bucket-Policy", + { + document: new PolicyDocument({ + statements: [ + new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject", + ], + resources: [ + S3ECGithubTestS3Bucket.bucketArn + "/*", // object-level permissions need this extra path + ], + }), + new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + "s3:ListBucket", + ], + resources: [ + S3ECGithubTestS3Bucket.bucketArn + ], + }), + ] + }), + } + ); + + // KMS key policy + const S3ECGithubKMSKeyPolicy = new ManagedPolicy( + this, + "S3EC-Python-Github-KMS-Key-Policy", + { + document: new PolicyDocument({ + statements: [ + new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + "kms:Decrypt", + "kms:GenerateDataKey", + "kms:GenerateDataKeyPair" + ], + resources: [ + S3ECGithubKMSKey.keyArn, + ] + }) + ] + }), + } + ) + + // IAM role + const GithubActionsPrincipal = new FederatedPrincipal( + "arn:aws:iam::" + this.account + ":oidc-provider/token.actions.githubusercontent.com", + { + "StringEquals": { + "token.actions.githubusercontent.com:aud": "sts.amazonaws.com" + }, + "StringLike": { + "token.actions.githubusercontent.com:sub": "repo:aws/amazon-s3-encryption-client-python:*" + } + }, + "sts:AssumeRoleWithWebIdentity" + ) + const S3ECGithubTestRole = new Role( + this, + "s3-github-test-role", + { + assumedBy: GithubActionsPrincipal, + roleName: "S3EC-Python-Github-test-role", + description: " Grant GitHub S3 put and get and KMS encrypt, decrypt, and generate access for testing", + path: "/", + managedPolicies: [S3ECGithubS3BucketPolicy, S3ECGithubKMSKeyPolicy] + } + ); + } +} diff --git a/cdk/package-lock.json b/cdk/package-lock.json new file mode 100644 index 00000000..4f44562c --- /dev/null +++ b/cdk/package-lock.json @@ -0,0 +1,4382 @@ +{ + "name": "cdk", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "cdk", + "version": "0.1.0", + "dependencies": { + "aws-cdk-lib": "2.92.0", + "constructs": "^10.0.0", + "source-map-support": "^0.5.21" + }, + "bin": { + "cdk": "bin/cdk.js" + }, + "devDependencies": { + "@types/jest": "^29.5.3", + "@types/node": "20.4.10", + "aws-cdk": "2.92.0", + "jest": "^29.6.2", + "ts-jest": "^29.1.1", + "ts-node": "^10.9.1", + "typescript": "~5.1.6" + } + }, + "node_modules/@ampproject/remapping": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", + "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@aws-cdk/asset-awscli-v1": { + "version": "2.2.247", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.247.tgz", + "integrity": "sha512-PGFzztdu5YozUgoUd8gq5qi1FR3EYMjNrl5JFrAlYh2w1PcTfExEwqDzZy9z6uzogEJKwQJDgyhWe+OcZzQqFg==", + "license": "Apache-2.0" + }, + "node_modules/@aws-cdk/asset-kubectl-v20": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.4.tgz", + "integrity": "sha512-Ps2MkmjYgMyflagqQ4dgTElc7Vwpqj8spw8dQVFiSeaaMPsuDSNsPax3/HjuDuwqsmLdaCZc6umlxYLpL0kYDA==", + "license": "Apache-2.0" + }, + "node_modules/@aws-cdk/asset-node-proxy-agent-v6": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.1.0.tgz", + "integrity": "sha512-7bY3J8GCVxLupn/kNmpPc5VJz8grx+4RKfnnJiO1LG+uxkZfANZG3RMHhE+qQxxwkyQ9/MfPtTpf748UhR425A==", + "license": "Apache-2.0" + }, + "node_modules/@babel/code-frame": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", + "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.27.1", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.28.0.tgz", + "integrity": "sha512-60X7qkglvrap8mn1lh2ebxXdZYtUcpd7gsmy9kLaBJ4i/WdY8PqTSdxyA8qraikqKQK5C1KRBKXqznrVapyNaw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.0.tgz", + "integrity": "sha512-UlLAnTPrFdNGoFtbSXwcGFQBtQZJCNjaN6hQNP3UPvuNXT1i82N26KL3dZeIpNalWywr9IuQuncaAfUaS1g6sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@ampproject/remapping": "^2.2.0", + "@babel/code-frame": "^7.27.1", + "@babel/generator": "^7.28.0", + "@babel/helper-compilation-targets": "^7.27.2", + "@babel/helper-module-transforms": "^7.27.3", + "@babel/helpers": "^7.27.6", + "@babel/parser": "^7.28.0", + "@babel/template": "^7.27.2", + "@babel/traverse": "^7.28.0", + "@babel/types": "^7.28.0", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.0.tgz", + "integrity": "sha512-lJjzvrbEeWrhB4P3QBsH7tey117PjLZnDbLiQEKjQ/fNJTjuq4HSqgFA+UNSwZT8D7dxxbnuSBMsa1lrWzKlQg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.28.0", + "@babel/types": "^7.28.0", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.27.2", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.27.2.tgz", + "integrity": "sha512-2+1thGUUWWjLTYTHZWK1n8Yga0ijBz1XAhUXcKy81rd5g6yh7hGqMp45v7cadSbEHc9G3OTv45SyneRN3ps4DQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.27.2", + "@babel/helper-validator-option": "^7.27.1", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", + "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz", + "integrity": "sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.27.1", + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.27.3", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.27.3.tgz", + "integrity": "sha512-dSOvYwvyLsWBeIRyOeHXp5vPj5l1I011r52FM1+r1jCERv+aFXYk4whgQccYEGYxK2H3ZAIA8nuPkQ0HaUo3qg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.27.1", + "@babel/helper-validator-identifier": "^7.27.1", + "@babel/traverse": "^7.27.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.27.1.tgz", + "integrity": "sha512-1gn1Up5YXka3YYAHGKpbideQ5Yjf1tDa9qYcgysz+cNCXukyLl6DjPXhD3VRwSb8c0J9tA4b2+rHEZtc6R0tlw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz", + "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", + "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.28.2", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.2.tgz", + "integrity": "sha512-/V9771t+EgXz62aCcyofnQhGM8DQACbRhvzKFsXKC9QM+5MadF8ZmIm0crDMaz3+o0h0zXfJnd4EhbYbxsrcFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.27.2", + "@babel/types": "^7.28.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.0.tgz", + "integrity": "sha512-jVZGvOxOuNSsuQuLRTh13nU0AogFlw32w/MT+LV6D3sP5WdbW61E77RnkbaO2dUvmPAYrBDJXGn5gGS6tH4j8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.0" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-bigint": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", + "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-properties": { + "version": "7.12.13", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", + "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.12.13" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-static-block": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", + "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-attributes": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.27.1.tgz", + "integrity": "sha512-oFT0FrKHgF53f4vOsZGi2Hh3I35PfSmVs4IBFLFj4dnafP+hIWDLg3VyKmUHfLoLHlyxY4C7DGtmHuJgn+IGww==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-meta": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", + "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-jsx": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.27.1.tgz", + "integrity": "sha512-y8YTNIeKoyhGd9O0Jiyzyyqk8gdjnumGTQPsz0xOZOQ2RmkVJeZ1vmmfIvFEKqucBG6axJGBZDE/7iI5suUI/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-logical-assignment-operators": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", + "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", + "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-numeric-separator": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", + "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-object-rest-spread": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", + "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-chaining": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", + "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-private-property-in-object": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", + "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-top-level-await": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", + "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-typescript": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.27.1.tgz", + "integrity": "sha512-xfYCBMxveHrRMnAWl1ZlPXOZjzkN82THFvLhQhFXFt81Z5HnN+EtUkZhv/zcKpmT3fzmWZB0ywiBrbC3vogbwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.27.2", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz", + "integrity": "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/parser": "^7.27.2", + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.0.tgz", + "integrity": "sha512-mGe7UK5wWyh0bKRfupsUchrQGqvDbZDbKJw+kcRGSmdHVYrv+ltd0pnpDTVpiTqnaBru9iEvA8pz8W46v0Amwg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/generator": "^7.28.0", + "@babel/helper-globals": "^7.28.0", + "@babel/parser": "^7.28.0", + "@babel/template": "^7.27.2", + "@babel/types": "^7.28.0", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.28.2", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.2.tgz", + "integrity": "sha512-ruv7Ae4J5dUYULmeXw1gmb7rYRz57OWCPM57pHojnLq/3Z1CK2lNSLTCVjxVk1F/TZHwOZZrOWi0ur95BbLxNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bcoe/v8-coverage": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", + "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@istanbuljs/load-nyc-config": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", + "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "camelcase": "^5.3.1", + "find-up": "^4.1.0", + "get-package-type": "^0.1.0", + "js-yaml": "^3.13.1", + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", + "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jest/console": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/console/-/console-29.7.0.tgz", + "integrity": "sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/core": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/core/-/core-29.7.0.tgz", + "integrity": "sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/reporters": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-changed-files": "^29.7.0", + "jest-config": "^29.7.0", + "jest-haste-map": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-resolve-dependencies": "^29.7.0", + "jest-runner": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "jest-watcher": "^29.7.0", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/@jest/environment": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.7.0.tgz", + "integrity": "sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-mock": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/expect": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-29.7.0.tgz", + "integrity": "sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "expect": "^29.7.0", + "jest-snapshot": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/expect-utils": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.7.0.tgz", + "integrity": "sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-get-type": "^29.6.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/fake-timers": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.7.0.tgz", + "integrity": "sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@sinonjs/fake-timers": "^10.0.2", + "@types/node": "*", + "jest-message-util": "^29.7.0", + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/globals": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.7.0.tgz", + "integrity": "sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/expect": "^29.7.0", + "@jest/types": "^29.6.3", + "jest-mock": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/reporters": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-29.7.0.tgz", + "integrity": "sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@bcoe/v8-coverage": "^0.2.3", + "@jest/console": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@jridgewell/trace-mapping": "^0.3.18", + "@types/node": "*", + "chalk": "^4.0.0", + "collect-v8-coverage": "^1.0.0", + "exit": "^0.1.2", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "istanbul-lib-coverage": "^3.0.0", + "istanbul-lib-instrument": "^6.0.0", + "istanbul-lib-report": "^3.0.0", + "istanbul-lib-source-maps": "^4.0.0", + "istanbul-reports": "^3.1.3", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "jest-worker": "^29.7.0", + "slash": "^3.0.0", + "string-length": "^4.0.1", + "strip-ansi": "^6.0.0", + "v8-to-istanbul": "^9.0.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/@jest/schemas": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", + "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.27.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/source-map": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-29.6.3.tgz", + "integrity": "sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.18", + "callsites": "^3.0.0", + "graceful-fs": "^4.2.9" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/test-result": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-29.7.0.tgz", + "integrity": "sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "collect-v8-coverage": "^1.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/test-sequencer": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-29.7.0.tgz", + "integrity": "sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/test-result": "^29.7.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/transform": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-29.7.0.tgz", + "integrity": "sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/types": "^29.6.3", + "@jridgewell/trace-mapping": "^0.3.18", + "babel-plugin-istanbul": "^6.1.1", + "chalk": "^4.0.0", + "convert-source-map": "^2.0.0", + "fast-json-stable-stringify": "^2.1.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "micromatch": "^4.0.4", + "pirates": "^4.0.4", + "slash": "^3.0.0", + "write-file-atomic": "^4.0.2" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/types": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz", + "integrity": "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.12", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.12.tgz", + "integrity": "sha512-OuLGC46TjB5BbN1dH8JULVVZY4WTdkF7tV9Ys6wLL1rubZnCMstOhNHueU5bLCrnRuDhKPDM4g6sw4Bel5Gzqg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.4.tgz", + "integrity": "sha512-VT2+G1VQs/9oz078bLrYbecdZKs912zQlkelYpuf+SXF+QvZDYJlbx/LSx+meSAwdDFnF8FVXW92AVjjkVmgFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.29", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.29.tgz", + "integrity": "sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@sinclair/typebox": { + "version": "0.27.8", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", + "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@sinonjs/commons": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", + "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/@sinonjs/fake-timers": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", + "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.0" + } + }, + "node_modules/@tsconfig/node10": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", + "integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/graceful-fs": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", + "integrity": "sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", + "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/istanbul-lib-report": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz", + "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/istanbul-lib-coverage": "*" + } + }, + "node_modules/@types/istanbul-reports": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz", + "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/istanbul-lib-report": "*" + } + }, + "node_modules/@types/jest": { + "version": "29.5.14", + "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.14.tgz", + "integrity": "sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "expect": "^29.0.0", + "pretty-format": "^29.0.0" + } + }, + "node_modules/@types/node": { + "version": "20.4.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.4.10.tgz", + "integrity": "sha512-vwzFiiy8Rn6E0MtA13/Cxxgpan/N6UeNYR9oUu6kuJWxu6zCk98trcDp8CBhbtaeuq9SykCmXkFr2lWLoPcvLg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/stack-utils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", + "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/yargs": { + "version": "17.0.33", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz", + "integrity": "sha512-WpxBCKWPLr4xSsHgz511rFJAM+wS28w2zEO1QDNY5zM/S8ok70NNfztH0xwhqKyaK0OHCbN98LDAZuy1ctxDkA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/yargs-parser": "*" + } + }, + "node_modules/@types/yargs-parser": { + "version": "21.0.3", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", + "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", + "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true, + "license": "MIT" + }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/aws-cdk": { + "version": "2.92.0", + "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.92.0.tgz", + "integrity": "sha512-9aAWJvZWSBJQxcsDopXYUAm6/pGz6vOQy2zfkn+YBuBkNelvW+ok15KPY4xn5m76tYnN79W03Gnfp/nxZUlcww==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "cdk": "bin/cdk" + }, + "engines": { + "node": ">= 14.15.0" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/aws-cdk-lib": { + "version": "2.92.0", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.92.0.tgz", + "integrity": "sha512-J+SUFSnOt9u2GbY5QIABgjGNiw8bL/v0S3zsPhhO1dVwK+G7oE+bhLcAi3iILrw2sIpirNWH9K3W0by9K+cyMw==", + "bundleDependencies": [ + "@balena/dockerignore", + "case", + "fs-extra", + "ignore", + "jsonschema", + "minimatch", + "punycode", + "semver", + "table", + "yaml" + ], + "license": "Apache-2.0", + "dependencies": { + "@aws-cdk/asset-awscli-v1": "^2.2.200", + "@aws-cdk/asset-kubectl-v20": "^2.1.2", + "@aws-cdk/asset-node-proxy-agent-v6": "^2.0.1", + "@balena/dockerignore": "^1.0.2", + "case": "1.6.3", + "fs-extra": "^11.1.1", + "ignore": "^5.2.4", + "jsonschema": "^1.4.1", + "minimatch": "^3.1.2", + "punycode": "^2.3.0", + "semver": "^7.5.4", + "table": "^6.8.1", + "yaml": "1.10.2" + }, + "engines": { + "node": ">= 14.15.0" + }, + "peerDependencies": { + "constructs": "^10.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/@balena/dockerignore": { + "version": "1.0.2", + "inBundle": true, + "license": "Apache-2.0" + }, + "node_modules/aws-cdk-lib/node_modules/ajv": { + "version": "8.12.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/aws-cdk-lib/node_modules/ansi-regex": { + "version": "5.0.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/ansi-styles": { + "version": "4.3.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/aws-cdk-lib/node_modules/astral-regex": { + "version": "2.0.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/balanced-match": { + "version": "1.0.2", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/brace-expansion": { + "version": "1.1.11", + "inBundle": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/aws-cdk-lib/node_modules/case": { + "version": "1.6.3", + "inBundle": true, + "license": "(MIT OR GPL-3.0-or-later)", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/color-convert": { + "version": "2.0.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/color-name": { + "version": "1.1.4", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/concat-map": { + "version": "0.0.1", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/emoji-regex": { + "version": "8.0.0", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/fast-deep-equal": { + "version": "3.1.3", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/fs-extra": { + "version": "11.1.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/aws-cdk-lib/node_modules/graceful-fs": { + "version": "4.2.11", + "inBundle": true, + "license": "ISC" + }, + "node_modules/aws-cdk-lib/node_modules/ignore": { + "version": "5.2.4", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/aws-cdk-lib/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/json-schema-traverse": { + "version": "1.0.0", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/jsonfile": { + "version": "6.1.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/aws-cdk-lib/node_modules/jsonschema": { + "version": "1.4.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/aws-cdk-lib/node_modules/lodash.truncate": { + "version": "4.4.2", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/lru-cache": { + "version": "6.0.0", + "inBundle": true, + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/aws-cdk-lib/node_modules/minimatch": { + "version": "3.1.2", + "inBundle": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/aws-cdk-lib/node_modules/punycode": { + "version": "2.3.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/aws-cdk-lib/node_modules/require-from-string": { + "version": "2.0.2", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/semver": { + "version": "7.5.4", + "inBundle": true, + "license": "ISC", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/aws-cdk-lib/node_modules/slice-ansi": { + "version": "4.0.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "astral-regex": "^2.0.0", + "is-fullwidth-code-point": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/aws-cdk-lib/node_modules/string-width": { + "version": "4.2.3", + "inBundle": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/strip-ansi": { + "version": "6.0.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/table": { + "version": "6.8.1", + "inBundle": true, + "license": "BSD-3-Clause", + "dependencies": { + "ajv": "^8.0.1", + "lodash.truncate": "^4.4.2", + "slice-ansi": "^4.0.0", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/universalify": { + "version": "2.0.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/uri-js": { + "version": "4.4.1", + "inBundle": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/yallist": { + "version": "4.0.0", + "inBundle": true, + "license": "ISC" + }, + "node_modules/aws-cdk-lib/node_modules/yaml": { + "version": "1.10.2", + "inBundle": true, + "license": "ISC", + "engines": { + "node": ">= 6" + } + }, + "node_modules/babel-jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", + "integrity": "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/transform": "^29.7.0", + "@types/babel__core": "^7.1.14", + "babel-plugin-istanbul": "^6.1.1", + "babel-preset-jest": "^29.6.3", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.8.0" + } + }, + "node_modules/babel-plugin-istanbul": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", + "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/helper-plugin-utils": "^7.0.0", + "@istanbuljs/load-nyc-config": "^1.0.0", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-instrument": "^5.0.4", + "test-exclude": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-istanbul/node_modules/istanbul-lib-instrument": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz", + "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/core": "^7.12.3", + "@babel/parser": "^7.14.7", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-jest-hoist": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz", + "integrity": "sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.3.3", + "@babel/types": "^7.3.3", + "@types/babel__core": "^7.1.14", + "@types/babel__traverse": "^7.0.6" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/babel-preset-current-node-syntax": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.2.0.tgz", + "integrity": "sha512-E/VlAEzRrsLEb2+dv8yp3bo4scof3l9nR4lrld+Iy5NyVqgVYUJnDAmunkhPMisRI32Qc4iRiz425d8vM++2fg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/plugin-syntax-async-generators": "^7.8.4", + "@babel/plugin-syntax-bigint": "^7.8.3", + "@babel/plugin-syntax-class-properties": "^7.12.13", + "@babel/plugin-syntax-class-static-block": "^7.14.5", + "@babel/plugin-syntax-import-attributes": "^7.24.7", + "@babel/plugin-syntax-import-meta": "^7.10.4", + "@babel/plugin-syntax-json-strings": "^7.8.3", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-syntax-numeric-separator": "^7.10.4", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.3", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5", + "@babel/plugin-syntax-top-level-await": "^7.14.5" + }, + "peerDependencies": { + "@babel/core": "^7.0.0 || ^8.0.0-0" + } + }, + "node_modules/babel-preset-jest": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz", + "integrity": "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA==", + "dev": true, + "license": "MIT", + "dependencies": { + "babel-plugin-jest-hoist": "^29.6.3", + "babel-preset-current-node-syntax": "^1.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.25.1", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.25.1.tgz", + "integrity": "sha512-KGj0KoOMXLpSNkkEI6Z6mShmQy0bc1I+T7K9N81k4WWMrfz+6fQ6es80B/YLAeRoKvjYE1YSHHOW1qe9xIVzHw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "caniuse-lite": "^1.0.30001726", + "electron-to-chromium": "^1.5.173", + "node-releases": "^2.0.19", + "update-browserslist-db": "^1.1.3" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/bs-logger": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/bs-logger/-/bs-logger-0.2.6.tgz", + "integrity": "sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-json-stable-stringify": "2.x" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/bser": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", + "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "node-int64": "^0.4.0" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "license": "MIT" + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001731", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001731.tgz", + "integrity": "sha512-lDdp2/wrOmTRWuoB5DpfNkC0rJDU8DqRa6nYL6HK6sytw70QMopt/NIc/9SM7ylItlBWfACXk0tEn37UWM/+mg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/char-regex": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", + "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/ci-info": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", + "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cjs-module-lexer": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", + "integrity": "sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", + "dev": true, + "license": "MIT", + "engines": { + "iojs": ">= 1.0.0", + "node": ">= 0.12.0" + } + }, + "node_modules/collect-v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.2.tgz", + "integrity": "sha512-lHl4d5/ONEbLlJvaJNtsF/Lz+WvB07u2ycqTYbdrq7UypDXailES4valYb2eWiJFxZlVmpGekfqoxQhzyFdT4Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "license": "MIT" + }, + "node_modules/constructs": { + "version": "10.4.2", + "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.4.2.tgz", + "integrity": "sha512-wsNxBlAott2qg8Zv87q3eYZYgheb9lchtBfjHzzLHtXbttwSrHPs1NNQbBrmbb1YZvYg2+Vh0Dor76w4mFxJkA==", + "license": "Apache-2.0" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/create-jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz", + "integrity": "sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-config": "^29.7.0", + "jest-util": "^29.7.0", + "prompts": "^2.0.1" + }, + "bin": { + "create-jest": "bin/create-jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/dedent": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.6.0.tgz", + "integrity": "sha512-F1Z+5UCFpmQUzJa11agbyPVMbpgT/qA3/SKyJ1jyBgm7dUcUEa8v9JwDkerSQXfakBwFljIxhOJqGkjUwZ9FSA==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "babel-plugin-macros": "^3.1.0" + }, + "peerDependenciesMeta": { + "babel-plugin-macros": { + "optional": true + } + } + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/detect-newline": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", + "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/diff-sequences": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", + "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.197", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.197.tgz", + "integrity": "sha512-m1xWB3g7vJ6asIFz+2pBUbq3uGmfmln1M9SSvBe4QIFWYrRHylP73zL/3nMjDmwz8V+1xAXQDfBd6+HPW0WvDQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/emittery": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", + "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sindresorhus/emittery?sponsor=1" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/exit": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", + "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==", + "dev": true, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/expect": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz", + "integrity": "sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/expect-utils": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fb-watchman": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", + "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bser": "2.1.1" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true, + "license": "ISC" + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/handlebars": { + "version": "4.7.8", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", + "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "minimist": "^1.2.5", + "neo-async": "^2.6.2", + "source-map": "^0.6.1", + "wordwrap": "^1.0.0" + }, + "bin": { + "handlebars": "bin/handlebars" + }, + "engines": { + "node": ">=0.4.7" + }, + "optionalDependencies": { + "uglify-js": "^3.1.4" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" + }, + "node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/import-local": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", + "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "pkg-dir": "^4.2.0", + "resolve-cwd": "^3.0.0" + }, + "bin": { + "import-local-fixture": "fixtures/cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "dev": true, + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "dev": true, + "license": "MIT" + }, + "node_modules/is-core-module": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "dev": true, + "license": "MIT", + "dependencies": { + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-generator-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz", + "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-instrument": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz", + "integrity": "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/core": "^7.23.9", + "@babel/parser": "^7.23.9", + "@istanbuljs/schema": "^0.1.3", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-instrument/node_modules/semver": { + "version": "7.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", + "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-source-maps": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz", + "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "debug": "^4.1.1", + "istanbul-lib-coverage": "^3.0.0", + "source-map": "^0.6.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.7.tgz", + "integrity": "sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", + "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/core": "^29.7.0", + "@jest/types": "^29.6.3", + "import-local": "^3.0.2", + "jest-cli": "^29.7.0" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/jest-changed-files": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-29.7.0.tgz", + "integrity": "sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w==", + "dev": true, + "license": "MIT", + "dependencies": { + "execa": "^5.0.0", + "jest-util": "^29.7.0", + "p-limit": "^3.1.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-circus": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-29.7.0.tgz", + "integrity": "sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/expect": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "co": "^4.6.0", + "dedent": "^1.0.0", + "is-generator-fn": "^2.0.0", + "jest-each": "^29.7.0", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "p-limit": "^3.1.0", + "pretty-format": "^29.7.0", + "pure-rand": "^6.0.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-cli": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-29.7.0.tgz", + "integrity": "sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/core": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "create-jest": "^29.7.0", + "exit": "^0.1.2", + "import-local": "^3.0.2", + "jest-config": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "yargs": "^17.3.1" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/jest-config": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-29.7.0.tgz", + "integrity": "sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/test-sequencer": "^29.7.0", + "@jest/types": "^29.6.3", + "babel-jest": "^29.7.0", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "deepmerge": "^4.2.2", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "jest-circus": "^29.7.0", + "jest-environment-node": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-runner": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "micromatch": "^4.0.4", + "parse-json": "^5.2.0", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@types/node": "*", + "ts-node": ">=9.0.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "ts-node": { + "optional": true + } + } + }, + "node_modules/jest-diff": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.7.0.tgz", + "integrity": "sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "diff-sequences": "^29.6.3", + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-docblock": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-29.7.0.tgz", + "integrity": "sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "detect-newline": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-each": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-29.7.0.tgz", + "integrity": "sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "jest-get-type": "^29.6.3", + "jest-util": "^29.7.0", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-environment-node": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-29.7.0.tgz", + "integrity": "sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-get-type": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz", + "integrity": "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-haste-map": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-29.7.0.tgz", + "integrity": "sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/graceful-fs": "^4.1.3", + "@types/node": "*", + "anymatch": "^3.0.3", + "fb-watchman": "^2.0.0", + "graceful-fs": "^4.2.9", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "jest-worker": "^29.7.0", + "micromatch": "^4.0.4", + "walker": "^1.0.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "optionalDependencies": { + "fsevents": "^2.3.2" + } + }, + "node_modules/jest-leak-detector": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz", + "integrity": "sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-matcher-utils": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz", + "integrity": "sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "jest-diff": "^29.7.0", + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-message-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.7.0.tgz", + "integrity": "sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "^29.6.3", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-mock": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-29.7.0.tgz", + "integrity": "sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-pnp-resolver": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", + "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + }, + "peerDependencies": { + "jest-resolve": "*" + }, + "peerDependenciesMeta": { + "jest-resolve": { + "optional": true + } + } + }, + "node_modules/jest-regex-util": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz", + "integrity": "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-resolve": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-29.7.0.tgz", + "integrity": "sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-pnp-resolver": "^1.2.2", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "resolve": "^1.20.0", + "resolve.exports": "^2.0.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-resolve-dependencies": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-29.7.0.tgz", + "integrity": "sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-regex-util": "^29.6.3", + "jest-snapshot": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-runner": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-29.7.0.tgz", + "integrity": "sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/environment": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "graceful-fs": "^4.2.9", + "jest-docblock": "^29.7.0", + "jest-environment-node": "^29.7.0", + "jest-haste-map": "^29.7.0", + "jest-leak-detector": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-resolve": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-util": "^29.7.0", + "jest-watcher": "^29.7.0", + "jest-worker": "^29.7.0", + "p-limit": "^3.1.0", + "source-map-support": "0.5.13" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-runner/node_modules/source-map-support": { + "version": "0.5.13", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", + "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/jest-runtime": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-29.7.0.tgz", + "integrity": "sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/fake-timers": "^29.7.0", + "@jest/globals": "^29.7.0", + "@jest/source-map": "^29.6.3", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "cjs-module-lexer": "^1.0.0", + "collect-v8-coverage": "^1.0.0", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-mock": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "slash": "^3.0.0", + "strip-bom": "^4.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-snapshot": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-29.7.0.tgz", + "integrity": "sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@babel/generator": "^7.7.2", + "@babel/plugin-syntax-jsx": "^7.7.2", + "@babel/plugin-syntax-typescript": "^7.7.2", + "@babel/types": "^7.3.3", + "@jest/expect-utils": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "babel-preset-current-node-syntax": "^1.0.0", + "chalk": "^4.0.0", + "expect": "^29.7.0", + "graceful-fs": "^4.2.9", + "jest-diff": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "natural-compare": "^1.4.0", + "pretty-format": "^29.7.0", + "semver": "^7.5.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-snapshot/node_modules/semver": { + "version": "7.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", + "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jest-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz", + "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "graceful-fs": "^4.2.9", + "picomatch": "^2.2.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-validate": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz", + "integrity": "sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "camelcase": "^6.2.0", + "chalk": "^4.0.0", + "jest-get-type": "^29.6.3", + "leven": "^3.1.0", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-validate/node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jest-watcher": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-29.7.0.tgz", + "integrity": "sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "jest-util": "^29.7.0", + "string-length": "^4.0.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-worker": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz", + "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "jest-util": "^29.7.0", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-worker/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true, + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/leven": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", + "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true, + "license": "MIT" + }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/lodash.memoize": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", + "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "7.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", + "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true, + "license": "ISC" + }, + "node_modules/makeerror": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", + "integrity": "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "tmpl": "1.0.5" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true, + "license": "MIT" + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-releases": { + "version": "2.0.19", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz", + "integrity": "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==", + "dev": true, + "license": "MIT" + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pirates": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", + "integrity": "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pretty-format": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", + "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "^29.6.3", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/pretty-format/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/prompts": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", + "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "kleur": "^3.0.3", + "sisteransi": "^1.0.5" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/pure-rand": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", + "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, + "node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "dev": true, + "license": "MIT" + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve": { + "version": "1.22.10", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz", + "integrity": "sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-core-module": "^2.16.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-cwd": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", + "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve.exports": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.3.tgz", + "integrity": "sha512-OcXjMsGdhL4XnbShKpAcSqPMzQoYkYyhbEaeSko47MjRP9NfEQMhZkXL1DoFlt9LWQn4YttrdnV6X2OiyzBi+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", + "dev": true, + "license": "MIT" + }, + "node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/stack-utils": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", + "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/string-length": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", + "integrity": "sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "char-regex": "^1.0.2", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-bom": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", + "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/test-exclude": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", + "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", + "dev": true, + "license": "ISC", + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^7.1.4", + "minimatch": "^3.0.4" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tmpl": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", + "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/ts-jest": { + "version": "29.4.1", + "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.4.1.tgz", + "integrity": "sha512-SaeUtjfpg9Uqu8IbeDKtdaS0g8lS6FT6OzM3ezrDfErPJPHNDo/Ey+VFGP1bQIDfagYDLyRpd7O15XpG1Es2Uw==", + "dev": true, + "license": "MIT", + "dependencies": { + "bs-logger": "^0.2.6", + "fast-json-stable-stringify": "^2.1.0", + "handlebars": "^4.7.8", + "json5": "^2.2.3", + "lodash.memoize": "^4.1.2", + "make-error": "^1.3.6", + "semver": "^7.7.2", + "type-fest": "^4.41.0", + "yargs-parser": "^21.1.1" + }, + "bin": { + "ts-jest": "cli.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || ^18.0.0 || >=20.0.0" + }, + "peerDependencies": { + "@babel/core": ">=7.0.0-beta.0 <8", + "@jest/transform": "^29.0.0 || ^30.0.0", + "@jest/types": "^29.0.0 || ^30.0.0", + "babel-jest": "^29.0.0 || ^30.0.0", + "jest": "^29.0.0 || ^30.0.0", + "jest-util": "^29.0.0 || ^30.0.0", + "typescript": ">=4.3 <6" + }, + "peerDependenciesMeta": { + "@babel/core": { + "optional": true + }, + "@jest/transform": { + "optional": true + }, + "@jest/types": { + "optional": true + }, + "babel-jest": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jest-util": { + "optional": true + } + } + }, + "node_modules/ts-jest/node_modules/semver": { + "version": "7.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", + "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/ts-jest/node_modules/type-fest": { + "version": "4.41.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", + "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, + "node_modules/type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/typescript": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.6.tgz", + "integrity": "sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/uglify-js": { + "version": "3.19.3", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", + "integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==", + "dev": true, + "license": "BSD-2-Clause", + "optional": true, + "bin": { + "uglifyjs": "bin/uglifyjs" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.3.tgz", + "integrity": "sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true, + "license": "MIT" + }, + "node_modules/v8-to-istanbul": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", + "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", + "dev": true, + "license": "ISC", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.12", + "@types/istanbul-lib-coverage": "^2.0.1", + "convert-source-map": "^2.0.0" + }, + "engines": { + "node": ">=10.12.0" + } + }, + "node_modules/walker": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", + "integrity": "sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "makeerror": "1.0.12" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/write-file-atomic": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", + "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", + "dev": true, + "license": "ISC", + "dependencies": { + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.7" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/cdk/package.json b/cdk/package.json new file mode 100644 index 00000000..f1e769db --- /dev/null +++ b/cdk/package.json @@ -0,0 +1,27 @@ +{ + "name": "cdk", + "version": "0.1.0", + "bin": { + "cdk": "bin/cdk.js" + }, + "scripts": { + "build": "tsc", + "watch": "tsc -w", + "test": "jest", + "cdk": "cdk" + }, + "devDependencies": { + "@types/jest": "^29.5.3", + "@types/node": "20.4.10", + "jest": "^29.6.2", + "ts-jest": "^29.1.1", + "aws-cdk": "2.92.0", + "ts-node": "^10.9.1", + "typescript": "~5.1.6" + }, + "dependencies": { + "aws-cdk-lib": "2.92.0", + "constructs": "^10.0.0", + "source-map-support": "^0.5.21" + } +} diff --git a/cdk/test/cdk.test.ts b/cdk/test/cdk.test.ts new file mode 100644 index 00000000..1e6b29c8 --- /dev/null +++ b/cdk/test/cdk.test.ts @@ -0,0 +1,17 @@ +// import * as cdk from 'aws-cdk-lib'; +// import { Template } from 'aws-cdk-lib/assertions'; +// import * as Cdk from '../lib/cdk-stack'; + +// example test. To run these tests, uncomment this file along with the +// example resource in lib/cdk-stack.ts +test('SQS Queue Created', () => { +// const app = new cdk.App(); +// // WHEN +// const stack = new Cdk.CdkStack(app, 'MyTestStack'); +// // THEN +// const template = Template.fromStack(stack); + +// template.hasResourceProperties('AWS::SQS::Queue', { +// VisibilityTimeout: 300 +// }); +}); diff --git a/cdk/tsconfig.json b/cdk/tsconfig.json new file mode 100644 index 00000000..aaa7dc51 --- /dev/null +++ b/cdk/tsconfig.json @@ -0,0 +1,31 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": [ + "es2020", + "dom" + ], + "declaration": true, + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "noImplicitThis": true, + "alwaysStrict": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": false, + "inlineSourceMap": true, + "inlineSources": true, + "experimentalDecorators": true, + "strictPropertyInitialization": false, + "typeRoots": [ + "./node_modules/@types" + ] + }, + "exclude": [ + "node_modules", + "cdk.out" + ] +} From df36ec12066abdd763c79b19f89f7307d8915435 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 6 Aug 2025 13:35:33 -0700 Subject: [PATCH 02/81] add basic S3EC implementation --- .gitignore | 12 + README.md | 17 +- SUPPORT_POLICY.rst | 29 ++ poetry.lock | 454 ++++++++++++++++++ pyproject.toml | 22 + src/s3_encryption/__init__.py | 84 ++++ src/s3_encryption/exceptions.py | 4 + src/s3_encryption/materials/__init__.py | 17 + .../materials/crypto_materials_manager.py | 62 +++ .../materials/encrypted_data_key.py | 20 + src/s3_encryption/materials/keyring.py | 100 ++++ src/s3_encryption/materials/kms_keyring.py | 117 +++++ src/s3_encryption/materials/materials.py | 59 +++ src/s3_encryption/metadata.py | 115 +++++ src/s3_encryption/pipelines.py | 150 ++++++ test/__init__.py | 2 + test/integration/__init__.py | 2 + test/integration/test_i_s3_encryption.py | 41 ++ test/test_encryption_materials.py | 57 +++ test/test_encryption_materials_integration.py | 85 ++++ test/test_metadata.py | 86 ++++ 21 files changed, 1519 insertions(+), 16 deletions(-) create mode 100644 .gitignore create mode 100644 SUPPORT_POLICY.rst create mode 100644 poetry.lock create mode 100644 pyproject.toml create mode 100644 src/s3_encryption/__init__.py create mode 100644 src/s3_encryption/exceptions.py create mode 100644 src/s3_encryption/materials/__init__.py create mode 100644 src/s3_encryption/materials/crypto_materials_manager.py create mode 100644 src/s3_encryption/materials/encrypted_data_key.py create mode 100644 src/s3_encryption/materials/keyring.py create mode 100644 src/s3_encryption/materials/kms_keyring.py create mode 100644 src/s3_encryption/materials/materials.py create mode 100644 src/s3_encryption/metadata.py create mode 100644 src/s3_encryption/pipelines.py create mode 100644 test/__init__.py create mode 100644 test/integration/__init__.py create mode 100644 test/integration/test_i_s3_encryption.py create mode 100644 test/test_encryption_materials.py create mode 100644 test/test_encryption_materials_integration.py create mode 100644 test/test_metadata.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..4b1c0c91 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +.idea +.vscode +# Exclude all pycache directories and bytecode +__pycache__/ +*.pyc +*.pyo +*.pyd + +# Distribution / packaging +dist/ +build/ +*.egg-info/ diff --git a/README.md b/README.md index 847260ca..2104b640 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,2 @@ -## My Project - -TODO: Fill this README out! - -Be sure to: - -* Change the title in this README -* Edit your repository description on GitHub - -## Security - -See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. - -## License - -This project is licensed under the Apache-2.0 License. +# Amazon S3 Encryption Client Python diff --git a/SUPPORT_POLICY.rst b/SUPPORT_POLICY.rst new file mode 100644 index 00000000..3eafd39b --- /dev/null +++ b/SUPPORT_POLICY.rst @@ -0,0 +1,29 @@ +Overview +======== +This page describes the support policy for the Amazon S3 Encryption Client. We regularly provide the Amazon S3 Encryption Client with updates that may contain support for new or updated APIs, new features, enhancements, bug fixes, security patches, or documentation updates. Updates may also address changes with dependencies, language runtimes, and operating systems. + +We recommend users to stay up-to-date with Amazon S3 Encryption Client releases to keep up with the latest features, security updates, and underlying dependencies. Continued use of an unsupported client version is not recommended and is done at the user’s discretion + + +Major Version Lifecycle +======================== +The Amazon S3 Encryption Client follows the same major version lifecycle as the AWS SDK. For details on this lifecycle, see `AWS SDKs and Tools Maintenance Policy`_. + +Version Support Matrix +====================== +This table describes the current support status of each major version of the Amazon S3 Encryption Client for Python. It also shows the next status each major version will transition to, and the date at which that transition will happen. + +.. list-table:: + :widths: 30 50 50 50 + :header-rows: 1 + + * - Major version + - Current status + - Next status + - Next status date + * - 3.x + - Pre-Release + - Generally Available + - + +.. _AWS SDKs and Tools Maintenance Policy: https://docs.aws.amazon.com/sdkref/latest/guide/maint-policy.html#version-life-cycle diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 00000000..1ab72864 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,454 @@ +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. + +[[package]] +name = "attrs" +version = "25.3.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.8" +files = [ + {file = "attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3"}, + {file = "attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b"}, +] + +[package.extras] +benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier"] +tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] + +[[package]] +name = "aws-cryptographic-material-providers" +version = "1.11.0" +description = "AWS Cryptographic Material Providers Library for Python" +optional = false +python-versions = "<4.0.0,>=3.11.0" +files = [ + {file = "aws_cryptographic_material_providers-1.11.0-py3-none-any.whl", hash = "sha256:9a9f0dca5b1902a4f16fb91cc1010dee74a721f84f411e81ffb4481fc0dd095f"}, + {file = "aws_cryptographic_material_providers-1.11.0.tar.gz", hash = "sha256:4ea5f9e5cc003e97d2ef98079dc25d8c49a0db01315ee887d19fd2f1c85ae9c3"}, +] + +[package.dependencies] +aws-cryptography-internal-dynamodb = "1.11.0" +aws-cryptography-internal-kms = "1.11.0" +aws-cryptography-internal-primitives = "1.11.0" +aws-cryptography-internal-standard-library = "1.11.0" + +[[package]] +name = "aws-cryptography-internal-dynamodb" +version = "1.11.0" +description = "" +optional = false +python-versions = "<4.0.0,>=3.11.0" +files = [ + {file = "aws_cryptography_internal_dynamodb-1.11.0-py3-none-any.whl", hash = "sha256:5a2da0ae6829d725f24018d001f4c733605f213820b723b6c75015843dc2427c"}, + {file = "aws_cryptography_internal_dynamodb-1.11.0.tar.gz", hash = "sha256:0800921ebb5dafc2853a2f5449f74aa03d24acd9ddb2ee58edca4002b97a5da5"}, +] + +[package.dependencies] +aws-cryptography-internal-standard-library = "1.11.0" +boto3 = ">=1.35.42,<2.0.0" + +[[package]] +name = "aws-cryptography-internal-kms" +version = "1.11.0" +description = "" +optional = false +python-versions = "<4.0.0,>=3.11.0" +files = [ + {file = "aws_cryptography_internal_kms-1.11.0-py3-none-any.whl", hash = "sha256:1c23cc8e970252fc7627868fc6b7a002400ec1d555ac29368e0eaddcceb07953"}, + {file = "aws_cryptography_internal_kms-1.11.0.tar.gz", hash = "sha256:a3ff5105b3e1c9d81e9698e0efc80de8a6bb8078b4512f9b39ed0f6161aae172"}, +] + +[package.dependencies] +aws-cryptography-internal-standard-library = "1.11.0" +boto3 = ">=1.35.42,<2.0.0" + +[[package]] +name = "aws-cryptography-internal-primitives" +version = "1.11.0" +description = "" +optional = false +python-versions = "<4.0.0,>=3.11.0" +files = [ + {file = "aws_cryptography_internal_primitives-1.11.0-py3-none-any.whl", hash = "sha256:84200885113f3534f4bff819ac1603c6d5c3bdd4d5c83a1b73ac2462cecec49b"}, + {file = "aws_cryptography_internal_primitives-1.11.0.tar.gz", hash = "sha256:9072af2c403b9e729dc767b44d1d642fa924a317a5bdbdffdf6dba0e93dc7996"}, +] + +[package.dependencies] +aws-cryptography-internal-standard-library = "1.11.0" +cryptography = ">=43.0.1,<46" + +[[package]] +name = "aws-cryptography-internal-standard-library" +version = "1.11.0" +description = "" +optional = false +python-versions = "<4.0.0,>=3.11.0" +files = [ + {file = "aws_cryptography_internal_standard_library-1.11.0-py3-none-any.whl", hash = "sha256:a2d5a4d8f70bce7242e8ebe06742223b8cd93253ed8081f44d7a8c1a086871e1"}, + {file = "aws_cryptography_internal_standard_library-1.11.0.tar.gz", hash = "sha256:36d82c6bc0361cf0ec3b7181804d375718f5c297949ddd902670f4452ecad3b0"}, +] + +[package.dependencies] +DafnyRuntimePython = "4.9.0" +pytz = ">=2023.3.post1,<2025.0.0" + +[[package]] +name = "boto3" +version = "1.39.14" +description = "The AWS SDK for Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "boto3-1.39.14-py3-none-any.whl", hash = "sha256:82c6868cad18c3bd4170915e9525f9af5f83e9779c528417f8863629558fc2d0"}, + {file = "boto3-1.39.14.tar.gz", hash = "sha256:fabb16360a93b449d5241006485bcc761c26694e75ac01009f4459f114acc06e"}, +] + +[package.dependencies] +botocore = ">=1.39.14,<1.40.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.13.0,<0.14.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.39.14" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.9" +files = [ + {file = "botocore-1.39.14-py3-none-any.whl", hash = "sha256:4ed551c77194167b7e8063f33059bc2f9b2ead0ed4ee33dc7857273648ed4349"}, + {file = "botocore-1.39.14.tar.gz", hash = "sha256:7fc44d4ad13b524e5d8a6296785776ef5898ac026ff74df9b35313831d507926"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} + +[package.extras] +crt = ["awscrt (==0.23.8)"] + +[[package]] +name = "cffi" +version = "1.17.1" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, + {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be"}, + {file = "cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c"}, + {file = "cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15"}, + {file = "cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401"}, + {file = "cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"}, + {file = "cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655"}, + {file = "cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0"}, + {file = "cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4"}, + {file = "cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93"}, + {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3"}, + {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8"}, + {file = "cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65"}, + {file = "cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903"}, + {file = "cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e"}, + {file = "cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd"}, + {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed"}, + {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9"}, + {file = "cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d"}, + {file = "cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a"}, + {file = "cffi-1.17.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1"}, + {file = "cffi-1.17.1-cp38-cp38-win32.whl", hash = "sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8"}, + {file = "cffi-1.17.1-cp38-cp38-win_amd64.whl", hash = "sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1"}, + {file = "cffi-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16"}, + {file = "cffi-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e"}, + {file = "cffi-1.17.1-cp39-cp39-win32.whl", hash = "sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7"}, + {file = "cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662"}, + {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "cryptography" +version = "43.0.3" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-43.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e1ce50266f4f70bf41a2c6dc4358afadae90e2a1e5342d3c08883df1675374f"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:443c4a81bb10daed9a8f334365fe52542771f25aedaf889fd323a853ce7377d6"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:74f57f24754fe349223792466a709f8e0c093205ff0dca557af51072ff47ab18"}, + {file = "cryptography-43.0.3-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9762ea51a8fc2a88b70cf2995e5675b38d93bf36bd67d91721c309df184f49bd"}, + {file = "cryptography-43.0.3-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:81ef806b1fef6b06dcebad789f988d3b37ccaee225695cf3e07648eee0fc6b73"}, + {file = "cryptography-43.0.3-cp37-abi3-win32.whl", hash = "sha256:cbeb489927bd7af4aa98d4b261af9a5bc025bd87f0e3547e11584be9e9427be2"}, + {file = "cryptography-43.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:f46304d6f0c6ab8e52770addfa2fc41e6629495548862279641972b6215451cd"}, + {file = "cryptography-43.0.3-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:8ac43ae87929a5982f5948ceda07001ee5e83227fd69cf55b109144938d96984"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:846da004a5804145a5f441b8530b4bf35afbf7da70f82409f151695b127213d5"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f996e7268af62598f2fc1204afa98a3b5712313a55c4c9d434aef49cadc91d4"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f7b178f11ed3664fd0e995a47ed2b5ff0a12d893e41dd0494f406d1cf555cab7"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:c2e6fc39c4ab499049df3bdf567f768a723a5e8464816e8f009f121a5a9f4405"}, + {file = "cryptography-43.0.3-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e1be4655c7ef6e1bbe6b5d0403526601323420bcf414598955968c9ef3eb7d16"}, + {file = "cryptography-43.0.3-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:df6b6c6d742395dd77a23ea3728ab62f98379eff8fb61be2744d4679ab678f73"}, + {file = "cryptography-43.0.3-cp39-abi3-win32.whl", hash = "sha256:d56e96520b1020449bbace2b78b603442e7e378a9b3bd68de65c782db1507995"}, + {file = "cryptography-43.0.3-cp39-abi3-win_amd64.whl", hash = "sha256:0c580952eef9bf68c4747774cde7ec1d85a6e61de97281f2dba83c7d2c806362"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:d03b5621a135bffecad2c73e9f4deb1a0f977b9a8ffe6f8e002bf6c9d07b918c"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a2a431ee15799d6db9fe80c82b055bae5a752bef645bba795e8e52687c69efe3"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:281c945d0e28c92ca5e5930664c1cefd85efe80e5c0d2bc58dd63383fda29f83"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f18c716be16bc1fea8e95def49edf46b82fccaa88587a45f8dc0ff6ab5d8e0a7"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:4a02ded6cd4f0a5562a8887df8b3bd14e822a90f97ac5e544c162899bc467664"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:53a583b6637ab4c4e3591a15bc9db855b8d9dee9a669b550f311480acab6eb08"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1ec0bcf7e17c0c5669d881b1cd38c4972fade441b27bda1051665faaa89bdcaa"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2ce6fae5bdad59577b44e4dfed356944fbf1d925269114c28be377692643b4ff"}, + {file = "cryptography-43.0.3.tar.gz", hash = "sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805"}, +] + +[package.dependencies] +cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] +nox = ["nox"] +pep8test = ["check-sdist", "click", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["certifi", "cryptography-vectors (==43.0.3)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + +[[package]] +name = "dafnyruntimepython" +version = "4.9.0" +description = "Dafny runtime for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "DafnyRuntimePython-4.9.0-py3-none-any.whl", hash = "sha256:c9cdcf127f5b6a4c6c9cf69016b9486318c3a6600e7f03fcbc621f6a5398479c"}, + {file = "dafnyruntimepython-4.9.0.tar.gz", hash = "sha256:03a4c2dbbe45c13dc2c7dbefad01812367b3bb217a14b4b848d7e94ef5c08cee"}, +] + +[[package]] +name = "iniconfig" +version = "2.1.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.8" +files = [ + {file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"}, + {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, +] + +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + +[[package]] +name = "packaging" +version = "25.0" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, + {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, + {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["coverage", "pytest", "pytest-benchmark"] + +[[package]] +name = "pycparser" +version = "2.22" +description = "C parser in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, + {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, +] + +[[package]] +name = "pygments" +version = "2.19.2" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, + {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, +] + +[package.extras] +windows-terminal = ["colorama (>=0.4.6)"] + +[[package]] +name = "pytest" +version = "8.4.1" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7"}, + {file = "pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c"}, +] + +[package.dependencies] +colorama = {version = ">=0.4", markers = "sys_platform == \"win32\""} +iniconfig = ">=1" +packaging = ">=20" +pluggy = ">=1.5,<2" +pygments = ">=2.7.2" + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2024.2" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, + {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, +] + +[[package]] +name = "s3transfer" +version = "0.13.1" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">=3.9" +files = [ + {file = "s3transfer-0.13.1-py3-none-any.whl", hash = "sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724"}, + {file = "s3transfer-0.13.1.tar.gz", hash = "sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf"}, +] + +[package.dependencies] +botocore = ">=1.37.4,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] + +[[package]] +name = "six" +version = "1.17.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, +] + +[[package]] +name = "urllib3" +version = "2.5.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.9" +files = [ + {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, + {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.11" +content-hash = "b1781f15e07cc26d093bb8ed243f85d126ac76a46954cc1d1ff29261f1db380c" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..fba01461 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,22 @@ +[tool.poetry] +name = "amazon-s3-encryption-client-python" +version = "0.1.0" +description = "This library provides an S3 client that supports client-side encryption." +authors = ["AWS Crypto Tools "] +license = "Apache-2.0" +readme = "README.md" +packages = [{include = "s3_encryption", from = "src"}] + +[tool.poetry.dependencies] +python = "^3.11" +boto3 = "^1.37.2" +# There is a newer version, but MPL wants this one. +cryptography = "^43.0.1" +aws-cryptographic-material-providers = "^1.7.4" +attrs = "^25.1.0" +pytest = "^8.4.1" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py new file mode 100644 index 00000000..99493f07 --- /dev/null +++ b/src/s3_encryption/__init__.py @@ -0,0 +1,84 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import io +from botocore.response import StreamingBody +from attrs import define, field +from .pipelines import PutEncryptedObjectPipeline, GetEncryptedObjectPipeline +from .materials.keyring import AbstractKeyring +from .materials.crypto_materials_manager import AbstractCryptoMaterialsManager, DefaultCryptoMaterialsManager +from .metadata import ObjectMetadata + + +@define +class S3EncryptionClientConfig(): + """ + Configuration object for the S3 Encryption Client + """ + keyring: AbstractKeyring + cmm: AbstractCryptoMaterialsManager = field() + @cmm.default + def _default_cmm_for_keyring(self): + return DefaultCryptoMaterialsManager(self.keyring) + + +@define +class S3EncryptionClient(): + wrapped_s3_client = field() + config: S3EncryptionClientConfig = field() + + # TODO: I don't know exactly how boto3 works, + # we maybe instead prefer only using kwargs? + # Do we need to provide specific arg overloads? + # TODO: rename Data-> Body to match boto + def put_object(self, Bucket, Key, Data, EncryptionContext=None, **kwargs): + # Create a pipeline for this operation + pipeline = PutEncryptedObjectPipeline(self.config.cmm) + + + # Encrypt the data using the pipeline + data_bytes = Data + # We probably just shouldn't support strings, use utf8 for now + if type(Data) == str: + data_bytes = Data.encode('utf-8') + encrypted_data, encryption_metadata = pipeline.encrypt(data_bytes, encryption_context=EncryptionContext) + + # Add encryption metadata to the request parameters + params = { + 'Bucket': Bucket, + 'Key': Key, + 'Body': encrypted_data, + **kwargs + } + + # Add encryption metadata to the parameters + if encryption_metadata: + # Merge any existing metadata with our encryption metadata + metadata = params.get('Metadata', {}) + metadata.update(encryption_metadata) + params['Metadata'] = metadata + + return self.wrapped_s3_client.put_object(**params) + + def get_object(self, EncryptionContext=None, **kwargs): + # try just straight kwargs + params = { + **kwargs + } + + # Get the encrypted object from S3 + response = self.wrapped_s3_client.get_object(**params) + + # Create a pipeline for this operation + pipeline = GetEncryptedObjectPipeline(self.config.cmm) + + # Decrypt the data using the pipeline + decrypted_data = pipeline.decrypt(response, EncryptionContext) #encrypted_data, encryption_metadata) + + # Create a new streaming body with the decrypted data + stream = io.BytesIO(decrypted_data) + streaming_body = StreamingBody(stream, len(decrypted_data)) + + # Update the response with the decrypted data + response['Body'] = streaming_body + + return response diff --git a/src/s3_encryption/exceptions.py b/src/s3_encryption/exceptions.py new file mode 100644 index 00000000..034b2bdf --- /dev/null +++ b/src/s3_encryption/exceptions.py @@ -0,0 +1,4 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +class S3EncryptionClientError(Exception): + """Exception class for S3 Encryption Client errors.""" diff --git a/src/s3_encryption/materials/__init__.py b/src/s3_encryption/materials/__init__.py new file mode 100644 index 00000000..79178052 --- /dev/null +++ b/src/s3_encryption/materials/__init__.py @@ -0,0 +1,17 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from .keyring import AbstractKeyring +from .kms_keyring import KmsKeyring +from .crypto_materials_manager import AbstractCryptoMaterialsManager, DefaultCryptoMaterialsManager +from .encrypted_data_key import EncryptedDataKey +from .materials import EncryptionMaterials + +__all__ = [ + 'AbstractKeyring', + 'KmsKeyring', + 'AbstractCryptoMaterialsManager', + 'DefaultCryptoMaterialsManager', + 'EncryptedDataKey', + 'EncryptionMaterials' +] diff --git a/src/s3_encryption/materials/crypto_materials_manager.py b/src/s3_encryption/materials/crypto_materials_manager.py new file mode 100644 index 00000000..33910d75 --- /dev/null +++ b/src/s3_encryption/materials/crypto_materials_manager.py @@ -0,0 +1,62 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from attrs import define +from .keyring import AbstractKeyring +from .materials import EncryptionMaterials +from typing import List, Dict, Any + +# API Stub for CMM +class AbstractCryptoMaterialsManager(): + def getEncryptionMaterials(self, encMatsRequest): + """ + Get encryption materials from the keyring. + + Args: + encMatsRequest (Dict[str, Any] or EncryptionMaterials): Request containing encryption parameters + + Returns: + EncryptionMaterials: The encryption materials + """ + raise NotImplementedError + + def decryptMaterials(self, decMatsRequest): + """ + Decrypt materials using the keyring. + + Args: + decMatsRequest (Dict[str, Any]): Request containing decryption parameters + + Returns: + Dict[str, Any]: The decryption materials + """ + raise NotImplementedError + +@define +class DefaultCryptoMaterialsManager(AbstractCryptoMaterialsManager): + keyring: AbstractKeyring + + def getEncryptionMaterials(self, encMatsRequest): + """ + Get encryption materials from the keyring. + + Args: + encMatsRequest (Dict[str, Any]): Request containing encryption parameters + + Returns: + EncryptionMaterials: The encryption materials + """ + # Convert dictionary to EncryptionMaterials if needed + if isinstance(encMatsRequest, dict): + materials = EncryptionMaterials( + encryption_context=encMatsRequest.get('encryption_context', {}) + ) + else: + materials = encMatsRequest + + return self.keyring.onEncrypt(materials) + + def decryptMaterials(self, decMatsRequest): + # TODO: Fill with defaults + stuff from decMatsRequest + materials = {**decMatsRequest} + encrypted_data_keys = decMatsRequest.get('encrypted_data_keys') + return self.keyring.onDecrypt(materials, encrypted_data_keys) diff --git a/src/s3_encryption/materials/encrypted_data_key.py b/src/s3_encryption/materials/encrypted_data_key.py new file mode 100644 index 00000000..72fbeae2 --- /dev/null +++ b/src/s3_encryption/materials/encrypted_data_key.py @@ -0,0 +1,20 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from attrs import define, field + +@define +class EncryptedDataKey: + """ + Class representing an encrypted data key. + + An encrypted data key contains information about the key provider + and the encrypted data key itself. + + Attributes: + key_provider_info (str): Information about the key provider + key_provider_id (bytes): Identifier for the key provider + encrypted_data_key (bytes): The encrypted data key + """ + key_provider_info: str = field() + key_provider_id: bytes = field() + encrypted_data_key: bytes = field() diff --git a/src/s3_encryption/materials/keyring.py b/src/s3_encryption/materials/keyring.py new file mode 100644 index 00000000..78a884e9 --- /dev/null +++ b/src/s3_encryption/materials/keyring.py @@ -0,0 +1,100 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from attrs import define, field +from ..exceptions import S3EncryptionClientError +from .materials import EncryptionMaterials + +@define +class AbstractKeyring(): + # Ideally, all keyrings would inherit this field. + # However, attrs doesn't allow us to set a default here, + # when inheriting keyrings have optional fields. + # Even without a default it doesn't seem to play nice with attrs. + #enableLegacyWrappingAlgorithms: bool = field(default=False) + + def onEncrypt(self, encMaterials): + """ + Process encryption materials. + + Args: + encMaterials (EncryptionMaterials): Encryption materials to process + + Returns: + EncryptionMaterials: The processed encryption materials + """ + raise NotImplementedError + + def onDecrypt(self, decMaterials, encrypted_data_keys=None): + """ + Decrypt one of the encrypted data keys and update decMaterials. + + Args: + decMaterials (dict): A dictionary containing decryption materials + encrypted_data_keys (List[EncryptedDataKey], optional): A list of encrypted data keys to try. + + Returns: + dict: The updated decMaterials with the plaintext data key (PDK) + """ + raise NotImplementedError + + +@define +class S3Keyring(AbstractKeyring): + """ + Base class for S3 encryption keyrings that provides common validation logic. + """ + # Ideally this would be set, but attrs doesn't play nice + # enable_legacy_wrapping_algorithms: bool = field(default=False) + + def onEncrypt(self, encMaterials): + """ + Validate encryption materials before encryption. + + Args: + encMaterials (EncryptionMaterials or dict): Encryption materials + + Returns: + EncryptionMaterials: The validated encryption materials + """ + # Convert dict to EncryptionMaterials if needed + if isinstance(encMaterials, dict): + encMaterials = EncryptionMaterials.from_dict(encMaterials) + + # Validate encryption materials + if not isinstance(encMaterials, EncryptionMaterials): + raise S3EncryptionClientError("Encryption materials must be an EncryptionMaterials instance or a dictionary") + + # Ensure encryption_context is a dictionary + if not isinstance(encMaterials.encryption_context, dict): + raise S3EncryptionClientError("Encryption context must be a dictionary") + + return encMaterials + + def onDecrypt(self, decMaterials, encrypted_data_keys=None): + """ + Validate decryption materials before decryption. + + Args: + decMaterials (dict): A dictionary containing decryption materials + encrypted_data_keys (List[EncryptedDataKey], optional): A list of encrypted data keys to try. + + Returns: + dict: The validated decryption materials + """ + # Validate decryption materials + if not isinstance(decMaterials, dict): + raise S3EncryptionClientError("Decryption materials must be a dictionary") + + # Validate encrypted_data_keys + if encrypted_data_keys is None or len(encrypted_data_keys) == 0: + raise S3EncryptionClientError("No encrypted data keys provided") + + # Ensure encryption contexts are dictionaries if present + if 'encryption_context_from_request' in decMaterials and not isinstance(decMaterials['encryption_context_from_request'], dict): + raise S3EncryptionClientError("Encryption context from request must be a dictionary") + + if 'encryption_context_stored' in decMaterials and not isinstance(decMaterials['encryption_context_stored'], dict): + raise S3EncryptionClientError("Stored encryption context must be a dictionary") + + return decMaterials diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py new file mode 100644 index 00000000..69f10700 --- /dev/null +++ b/src/s3_encryption/materials/kms_keyring.py @@ -0,0 +1,117 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from .keyring import S3Keyring +from .encrypted_data_key import EncryptedDataKey +from .materials import EncryptionMaterials +from ..exceptions import S3EncryptionClientError +from attrs import define, field + +KMS_CONTEXT_DEFAULT_KEY = "aws:x-amz-cek-alg" +KMS_V1_DEFAULT_KEY = "kms_cmk_id" + +@define +class KmsKeyring(S3Keyring): + kms_client = field() + kms_key_id: str = field() + enable_legacy_wrapping_algorithms: bool = field(default=False) + + def onEncrypt(self, encMaterials): + """ + Process encryption materials using KMS. + + Args: + encMaterials (EncryptionMaterials): Encryption materials to process + + Returns: + EncryptionMaterials: The processed encryption materials with KMS-generated keys + """ + try: + # Call parent class validation + encMaterials = super().onEncrypt(encMaterials) + + # Add default encryption context + encryption_context = encMaterials.encryption_context + encryption_context["aws:x-amz-cek-alg"] = "AES/GCM/NoPadding" + + response = self.kms_client.generate_data_key( + KeyId = self.kms_key_id, + KeySpec = 'AES_256', + EncryptionContext = encryption_context + ) + # Create an EncryptedDataKey instance + encrypted_data_key = EncryptedDataKey( + key_provider_id=b'S3Keyring', + key_provider_info="kms+context", + encrypted_data_key=response['CiphertextBlob'] + ) + encMaterials.encrypted_data_key = encrypted_data_key + encMaterials.plaintext_data_key = response['Plaintext'] + return encMaterials + except Exception as e: + raise + + def onDecrypt(self, decMaterials, encrypted_data_keys=None): + """ + Decrypt one of the encrypted data keys and update decMaterials. + + Args: + decMaterials (dict): A dictionary containing decryption materials + encrypted_data_keys (List[EncryptedDataKey], optional): A list of encrypted data keys to try. + + Returns: + dict: The updated decMaterials with the plaintext data key (PDK) + """ + try: + # Call parent class validation + decMaterials = super().onDecrypt(decMaterials, encrypted_data_keys) + + # Handle both single EDK (backward compatibility) and list of EDKs + edks = encrypted_data_keys + + # Try to decrypt each EDK until one succeeds + # TODO: probably just enforce |EDKs| == 1 and remove loop + last_exception = None + for edk in edks: + try: + edk_bytes = edk.encrypted_data_key + if edk.key_provider_info == "kms+context": + encryption_context_from_request = decMaterials.get('encryption_context_from_request', {}) + encryption_context_stored = decMaterials.get('encryption_context_stored', {}) + + # Default EC MUST NOT be passed in via request + if KMS_CONTEXT_DEFAULT_KEY in encryption_context_from_request: + raise S3EncryptionClientError(f"{KMS_CONTEXT_DEFAULT_KEY} is a reserved key for the S3 encryption client") + + # The stored EC, minus default key/values, MUST match provided EC + encryption_context_stored_copy = encryption_context_stored.copy() + encryption_context_stored_copy.pop(KMS_V1_DEFAULT_KEY, None) + encryption_context_stored_copy.pop(KMS_CONTEXT_DEFAULT_KEY, None) + if encryption_context_stored_copy != encryption_context_from_request: + # TODO: modeled error + raise S3EncryptionClientError("Provided encryption context does not match information retrieved from S3") + + # Update decMaterials with the modified encryption context + elif edk.key_provider_info == "kms": + if not self.enable_legacy_wrapping_algorithms: + raise S3EncryptionClientError(f"Enable legacy wrapping algorithms to use legacy key wrapping algorithm: {edk.key_provider_info}") + else: + raise S3EncryptionClientError(f"{edk.key_provider_info} is not a valid key wrapping algorithm!") + + response = self.kms_client.decrypt( + KeyId = self.kms_key_id, + CiphertextBlob = edk_bytes, + EncryptionContext = decMaterials['encryption_context_stored'] + ) + decMaterials['PDK'] = response['Plaintext'] + return decMaterials + except Exception as e: + last_exception = e + continue + + # If we get here, none of the EDKs could be decrypted + if last_exception: + raise last_exception + else: + raise S3EncryptionClientError("Failed to decrypt any of the encrypted data keys") + except Exception as e: + raise diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py new file mode 100644 index 00000000..32b0b880 --- /dev/null +++ b/src/s3_encryption/materials/materials.py @@ -0,0 +1,59 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from attrs import define, field +from typing import Optional, Dict, Any +from .encrypted_data_key import EncryptedDataKey + +@define +class EncryptionMaterials: + """ + Class representing encryption materials for S3 encryption. + + This class provides a structured way to handle encryption materials + with fields corresponding to the data needed for encryption operations. + + Attributes: + encryption_context (Dict[str, str]): Context information for encryption + encrypted_data_key (Optional[EncryptedDataKey]): The encrypted data key + plaintext_data_key (Optional[bytes]): The plaintext data key (PDK) + """ + encryption_context: Dict[str, str] = field(factory=dict) + encrypted_data_key: Optional[EncryptedDataKey] = field(default=None) + plaintext_data_key: Optional[bytes] = field(default=None) + + @classmethod + def from_dict(cls, materials_dict: Dict[str, Any]) -> 'EncryptionMaterials': + """ + Create an EncryptionMaterials instance from a dictionary. + + Args: + materials_dict (Dict[str, Any]): Dictionary containing encryption materials + + Returns: + EncryptionMaterials: A new instance with fields populated from the dictionary + """ + return cls( + encryption_context=materials_dict.get('encryption_context', {}), + encrypted_data_key=materials_dict.get('encrypted_data_key'), + plaintext_data_key=materials_dict.get('PDK') + ) + + def to_dict(self) -> Dict[str, Any]: + """ + Convert the EncryptionMaterials instance to a dictionary. + + Returns: + Dict[str, Any]: Dictionary containing encryption materials + """ + result = {} + + if self.encryption_context: + result['encryption_context'] = self.encryption_context + + if self.encrypted_data_key is not None: + result['encrypted_data_key'] = self.encrypted_data_key + + if self.plaintext_data_key is not None: + result['PDK'] = self.plaintext_data_key + + return result diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py new file mode 100644 index 00000000..4894792a --- /dev/null +++ b/src/s3_encryption/metadata.py @@ -0,0 +1,115 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import json +from attrs import define, field +from typing import Optional, Dict, Any + + +@define +class ObjectMetadata: + """ + Class representing metadata for encrypted S3 objects. + + This class provides a structured way to handle encryption metadata + with fields corresponding to standard S3 encryption headers. + + All fields are optional and correspond to the following S3 encryption headers: + - encrypted_data_key_v1: The encrypted data key (legacy format) + - encrypted_data_key_v2: The encrypted data key (current format) + - encrypted_data_key_algorithm: The algorithm used to encrypt the data key (e.g. AES/GCM or kms+context) + - encrypted_data_key_context: The encryption context used for the data key + - content_iv: The initialization vector used for content encryption + - content_cipher: The cipher algorithm used for content encryption (e.g. AES/GCM/NoPadding) + - content_cipher_tag_length: The length of the authentication tag + - instruction_file: Marker for instruction files + """ + # The encrypted data key (legacy format) + encrypted_data_key_v1: Optional[str] = field(default=None) + # The encrypted data key (current format) + encrypted_data_key_v2: Optional[str] = field(default=None) + # The algorithm used to encrypt the data key (e.g. AES/GCM or kms+context) + encrypted_data_key_algorithm: Optional[str] = field(default=None) + # The encryption context used for the data key + encrypted_data_key_context: Optional[dict] = field(default=None) + # The initialization vector used for content encryption + content_iv: Optional[str] = field(default=None) + # The cipher algorithm used for content encryption (e.g. AES/GCM/NoPadding) + content_cipher: Optional[str] = field(default=None) + # The length of the authentication tag + content_cipher_tag_length: Optional[str] = field(default="128") + # Marker for instruction files + instruction_file: Optional[str] = field(default=None) + + # Constants for metadata keys + ENCRYPTED_DATA_KEY_V1 = "x-amz-key" + ENCRYPTED_DATA_KEY_V2 = "x-amz-key-v2" + ENCRYPTED_DATA_KEY_ALGORITHM = "x-amz-wrap-alg" + ENCRYPTED_DATA_KEY_CONTEXT = "x-amz-matdesc" + CONTENT_IV = "x-amz-iv" + CONTENT_CIPHER = "x-amz-cek-alg" + CONTENT_CIPHER_TAG_LENGTH = "x-amz-tag-len" + INSTRUCTION_FILE = "x-amz-crypto-instr-file" + + @classmethod + def from_dict(cls, metadata_dict: Dict[str, Any]) -> 'ObjectMetadata': + """ + Create an ObjectMetadata instance from a dictionary. + + Args: + metadata_dict (Dict[str, Any]): Dictionary containing metadata keys and values + + Returns: + ObjectMetadata: A new instance with fields populated from the dictionary + """ + # Parse the encryption context if present + encryption_context = None + if cls.ENCRYPTED_DATA_KEY_CONTEXT in metadata_dict: + context_str = metadata_dict.get(cls.ENCRYPTED_DATA_KEY_CONTEXT) + if context_str is not None: + encryption_context = json.loads(context_str) + + return cls( + encrypted_data_key_v1=metadata_dict.get(cls.ENCRYPTED_DATA_KEY_V1), + encrypted_data_key_v2=metadata_dict.get(cls.ENCRYPTED_DATA_KEY_V2), + encrypted_data_key_algorithm=metadata_dict.get(cls.ENCRYPTED_DATA_KEY_ALGORITHM), + encrypted_data_key_context=encryption_context, + content_iv=metadata_dict.get(cls.CONTENT_IV), + content_cipher=metadata_dict.get(cls.CONTENT_CIPHER), + content_cipher_tag_length=metadata_dict.get(cls.CONTENT_CIPHER_TAG_LENGTH), + instruction_file=metadata_dict.get(cls.INSTRUCTION_FILE) + ) + + def to_dict(self) -> Dict[str, str]: + """ + Convert the ObjectMetadata instance to a dictionary. + + Returns: + Dict[str, str]: Dictionary containing non-None metadata values + """ + result = {} + + if self.encrypted_data_key_v1 is not None: + result[self.ENCRYPTED_DATA_KEY_V1] = self.encrypted_data_key_v1 + + if self.encrypted_data_key_v2 is not None: + result[self.ENCRYPTED_DATA_KEY_V2] = self.encrypted_data_key_v2 + + if self.encrypted_data_key_algorithm is not None: + result[self.ENCRYPTED_DATA_KEY_ALGORITHM] = self.encrypted_data_key_algorithm + + if self.encrypted_data_key_context is not None: + result[self.ENCRYPTED_DATA_KEY_CONTEXT] = json.dumps(self.encrypted_data_key_context) + + if self.content_iv is not None: + result[self.CONTENT_IV] = self.content_iv + + if self.content_cipher is not None: + result[self.CONTENT_CIPHER] = self.content_cipher + + if self.content_cipher_tag_length is not None: + result[self.CONTENT_CIPHER_TAG_LENGTH] = self.content_cipher_tag_length + + if self.instruction_file is not None: + result[self.INSTRUCTION_FILE] = self.instruction_file + + return result diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py new file mode 100644 index 00000000..7ddee40c --- /dev/null +++ b/src/s3_encryption/pipelines.py @@ -0,0 +1,150 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from attrs import define, field +import os +import base64 +from .materials.crypto_materials_manager import AbstractCryptoMaterialsManager +from .materials.encrypted_data_key import EncryptedDataKey +from .materials.materials import EncryptionMaterials +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from .metadata import ObjectMetadata + +@define +class PutEncryptedObjectPipeline: + """ + Pipeline for encrypting objects before they are put into S3. + + This pipeline handles only the encryption process for S3 objects. + The actual S3 API calls are handled by the S3EncryptionClient. + """ + cmm: AbstractCryptoMaterialsManager = field() + + def encrypt(self, plaintext, encryption_context=None): + """ + Encrypt the data before it is stored in S3. + + Args: + data (bytes or str): The data to be encrypted + encryption_context (dict, optional): Additional context for encryption + + Returns: + bytes: The encrypted data + dict: Metadata about the encryption to be stored with the object + """ + # Create encryption materials request with encryption context + enc_mats_request = EncryptionMaterials( + encryption_context={} if encryption_context is None else encryption_context + ) + + # Get encryption materials from the crypto materials manager + enc_mats = self.cmm.getEncryptionMaterials(enc_mats_request) + + # Generate initialization vector + iv = os.urandom(12) + + # Encrypt the data + if enc_mats.plaintext_data_key is None: + raise RuntimeError("No plaintext data key found!") + + aesgcm = AESGCM(enc_mats.plaintext_data_key) + ciphertext = aesgcm.encrypt( + nonce=iv, + data=plaintext, + associated_data=None + ) + encrypted_data = ciphertext + b64_iv = base64.b64encode(iv).decode('utf-8') + + # Get the encrypted data key + if enc_mats.encrypted_data_key is None: + raise RuntimeError("No encrypted data key found!") + + edk_bytes = enc_mats.encrypted_data_key.encrypted_data_key + b64_edk = base64.b64encode(edk_bytes).decode('utf-8') + + # Create metadata using the ObjectMetadata class + metadata = ObjectMetadata( + encrypted_data_key_v2=b64_edk, + encrypted_data_key_algorithm="kms+context", + content_iv=b64_iv, + content_cipher="AES/GCM/NoPadding", + encrypted_data_key_context=enc_mats.encryption_context + ) + + # Convert to dictionary for storage in S3 metadata + encryption_metadata = metadata.to_dict() + + return encrypted_data, encryption_metadata + + +@define +class GetEncryptedObjectPipeline: + """ + Pipeline for decrypting objects after they are retrieved from S3. + + This pipeline handles only the decryption process for S3 objects. + The actual S3 API calls are handled by the S3EncryptionClient. + """ + cmm: AbstractCryptoMaterialsManager = field() + + def decrypt(self, response, encryption_context={}): + """ + Decrypt the data after it is retrieved from S3. + + Args: + encrypted_data (bytes): The encrypted data retrieved from S3 + encryption_metadata (dict, optional): Metadata about the encryption + + Returns: + bytes or str: The decrypted data + """ + # Convert the metadata dictionary to an ObjectMetadata instance + encrypted_data = response.get('Body').read() + encryption_metadata = response.get('Metadata', {}) + metadata = ObjectMetadata.from_dict(encryption_metadata) + + iv_b64 = metadata.content_iv + edk_b64 = metadata.encrypted_data_key_v2 + + # TODO: probably move this to ObjectMetadata + iv_bytes = base64.b64decode(iv_b64) + + # Create a list of encrypted data keys to try + encrypted_data_keys = [] + # Create an instance of EncryptedDataKey + if edk_b64: + edk_bytes = base64.b64decode(edk_b64) + encrypted_data_key = EncryptedDataKey( + key_provider_id=b'S3Keyring', + key_provider_info=metadata.encrypted_data_key_algorithm, + encrypted_data_key=edk_bytes + ) + encrypted_data_keys.append(encrypted_data_key) + + # Also check for legacy encrypted data key (v1) if available + if metadata.encrypted_data_key_v1: + legacy_edk_bytes = base64.b64decode(metadata.encrypted_data_key_v1) + legacy_encrypted_data_key = EncryptedDataKey( + key_provider_id=b'S3Keyring', + key_provider_info=metadata.encrypted_data_key_algorithm, + encrypted_data_key=legacy_edk_bytes + ) + encrypted_data_keys.append(legacy_encrypted_data_key) + + dec_mat_req = { + "iv": iv_bytes, + "encrypted_data_keys": encrypted_data_keys, + "encryption_context_stored": metadata.encrypted_data_key_context, + "encryption_context_from_request": encryption_context + } + dec_mats = self.cmm.decryptMaterials(dec_mat_req) + + aesgcm = AESGCM(dec_mats['PDK']) + + plaintext = aesgcm.decrypt( + nonce=iv_bytes, + data=encrypted_data, + associated_data=None + ) + + return plaintext diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 00000000..f94fd12a --- /dev/null +++ b/test/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/test/integration/__init__.py b/test/integration/__init__.py new file mode 100644 index 00000000..f94fd12a --- /dev/null +++ b/test/integration/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py new file mode 100644 index 00000000..981fbd3c --- /dev/null +++ b/test/integration/test_i_s3_encryption.py @@ -0,0 +1,41 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import boto3 +from datetime import datetime +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.materials.kms_keyring import KmsKeyring + +bucket = "s3-ec-python-v3-test" + +def test_simple_roundtrip(): + key = "simple-rt" + key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + + data = "test input for simple v3 round trip" + + kms_key_id = "arn:aws:kms:us-east-2:657301468084:key/1f469b1a-5cfa-4879-9bdf-27b3abd9b8d5" + kms_client = boto3.client("kms", region_name="us-east-2") + + keyring = KmsKeyring(kms_client, kms_key_id) + + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + s3ec.put_object(Bucket=bucket, Key=key, Data=data) + print("put object success!") + get_req = { + 'Bucket': bucket, + 'Key': key + } + response = s3ec.get_object(**get_req) + output = response['Body'].read().decode('utf-8') + print("get succeeded!") + print(response) + if output != data: + print("Uh oh! Input and output don't match!") + print("Input:") + print(input) + print("Output:") + print(output) + else: + print("Success!") \ No newline at end of file diff --git a/test/test_encryption_materials.py b/test/test_encryption_materials.py new file mode 100644 index 00000000..b1e6b653 --- /dev/null +++ b/test/test_encryption_materials.py @@ -0,0 +1,57 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import unittest +from src.s3_encryption.materials.materials import EncryptionMaterials +from src.s3_encryption.materials.encrypted_data_key import EncryptedDataKey + +class TestEncryptionMaterials(unittest.TestCase): + def test_create_encryption_materials(self): + """Test creating an EncryptionMaterials instance.""" + materials = EncryptionMaterials() + self.assertEqual(materials.encryption_context, {}) + self.assertIsNone(materials.encrypted_data_key) + self.assertIsNone(materials.plaintext_data_key) + + def test_create_with_encryption_context(self): + """Test creating an EncryptionMaterials instance with an encryption context.""" + encryption_context = {"key1": "value1", "key2": "value2"} + materials = EncryptionMaterials(encryption_context=encryption_context) + self.assertEqual(materials.encryption_context, encryption_context) + + def test_from_dict(self): + """Test creating an EncryptionMaterials instance from a dictionary.""" + edk = EncryptedDataKey( + key_provider_id=b'S3Keyring', + key_provider_info="kms+context", + encrypted_data_key=b'encrypted-data-key' + ) + materials_dict = { + 'encryption_context': {"key1": "value1"}, + 'encrypted_data_key': edk, + 'PDK': b'plaintext-data-key' + } + materials = EncryptionMaterials.from_dict(materials_dict) + self.assertEqual(materials.encryption_context, {"key1": "value1"}) + self.assertEqual(materials.encrypted_data_key, edk) + self.assertEqual(materials.plaintext_data_key, b'plaintext-data-key') + + def test_to_dict(self): + """Test converting an EncryptionMaterials instance to a dictionary.""" + edk = EncryptedDataKey( + key_provider_id=b'S3Keyring', + key_provider_info="kms+context", + encrypted_data_key=b'encrypted-data-key' + ) + materials = EncryptionMaterials( + encryption_context={"key1": "value1"}, + encrypted_data_key=edk, + plaintext_data_key=b'plaintext-data-key' + ) + materials_dict = materials.to_dict() + self.assertEqual(materials_dict['encryption_context'], {"key1": "value1"}) + self.assertEqual(materials_dict['encrypted_data_key'], edk) + self.assertEqual(materials_dict['PDK'], b'plaintext-data-key') + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_encryption_materials_integration.py b/test/test_encryption_materials_integration.py new file mode 100644 index 00000000..7082bb8f --- /dev/null +++ b/test/test_encryption_materials_integration.py @@ -0,0 +1,85 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import unittest +from unittest.mock import MagicMock, patch +from src.s3_encryption.materials.materials import EncryptionMaterials +from src.s3_encryption.materials.encrypted_data_key import EncryptedDataKey +from src.s3_encryption.materials.keyring import S3Keyring +from src.s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager + +class TestEncryptionMaterialsIntegration(unittest.TestCase): + def test_keyring_onEncrypt(self): + """Test that S3Keyring.onEncrypt properly handles EncryptionMaterials.""" + # Create a keyring + keyring = S3Keyring() + + # Create encryption materials + materials = EncryptionMaterials( + encryption_context={"key1": "value1"} + ) + + # Call onEncrypt + result = keyring.onEncrypt(materials) + + # Verify the result is an EncryptionMaterials instance + self.assertIsInstance(result, EncryptionMaterials) + self.assertEqual(result.encryption_context, {"key1": "value1"}) + + def test_cmm_getEncryptionMaterials_with_dict(self): + """Test that DefaultCryptoMaterialsManager.getEncryptionMaterials properly handles dictionary input.""" + # Create a mock keyring + keyring = MagicMock() + keyring.onEncrypt.return_value = EncryptionMaterials( + encryption_context={"key1": "value1"}, + encrypted_data_key=EncryptedDataKey( + key_provider_id=b'S3Keyring', + key_provider_info="kms+context", + encrypted_data_key=b'encrypted-data-key' + ), + plaintext_data_key=b'plaintext-data-key' + ) + + # Create a CMM + cmm = DefaultCryptoMaterialsManager(keyring=keyring) + + # Call getEncryptionMaterials with a dictionary + result = cmm.getEncryptionMaterials({"encryption_context": {"key1": "value1"}}) + + # Verify the result is an EncryptionMaterials instance + self.assertIsInstance(result, EncryptionMaterials) + self.assertEqual(result.encryption_context, {"key1": "value1"}) + self.assertIsNotNone(result.encrypted_data_key) + self.assertIsNotNone(result.plaintext_data_key) + + def test_cmm_getEncryptionMaterials_with_materials(self): + """Test that DefaultCryptoMaterialsManager.getEncryptionMaterials properly handles EncryptionMaterials input.""" + # Create a mock keyring + keyring = MagicMock() + keyring.onEncrypt.return_value = EncryptionMaterials( + encryption_context={"key1": "value1"}, + encrypted_data_key=EncryptedDataKey( + key_provider_id=b'S3Keyring', + key_provider_info="kms+context", + encrypted_data_key=b'encrypted-data-key' + ), + plaintext_data_key=b'plaintext-data-key' + ) + + # Create a CMM + cmm = DefaultCryptoMaterialsManager(keyring=keyring) + + # Call getEncryptionMaterials with an EncryptionMaterials instance + materials = EncryptionMaterials( + encryption_context={"key1": "value1"} + ) + result = cmm.getEncryptionMaterials(materials) + + # Verify the result is an EncryptionMaterials instance + self.assertIsInstance(result, EncryptionMaterials) + self.assertEqual(result.encryption_context, {"key1": "value1"}) + self.assertIsNotNone(result.encrypted_data_key) + self.assertIsNotNone(result.plaintext_data_key) + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_metadata.py b/test/test_metadata.py new file mode 100644 index 00000000..40a813aa --- /dev/null +++ b/test/test_metadata.py @@ -0,0 +1,86 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import unittest +import sys +import os + +# Add the src directory to the Python path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../src'))) + +from s3_encryption.metadata import ObjectMetadata + + +class TestObjectMetadata(unittest.TestCase): + def test_from_dict(self): + # Create a metadata dictionary + metadata_dict = { + "x-amz-key-v2": "encrypted-key-data", + "x-amz-wrap-alg": "kms+context", + "x-amz-iv": "base64-encoded-iv", + "x-amz-cek-alg": "AES/GCM/NoPadding" + } + + # Create an ObjectMetadata instance from the dictionary + metadata = ObjectMetadata.from_dict(metadata_dict) + + # Verify that the fields were populated correctly + self.assertEqual(metadata.encrypted_data_key_v2, "encrypted-key-data") + self.assertEqual(metadata.encrypted_data_key_algorithm, "kms+context") + self.assertEqual(metadata.content_iv, "base64-encoded-iv") + self.assertEqual(metadata.content_cipher, "AES/GCM/NoPadding") + + # Verify that fields not in the dictionary are None + self.assertIsNone(metadata.encrypted_data_key_v1) + self.assertIsNone(metadata.encrypted_data_key_context) + # Note: content_cipher_tag_length is None because it's not in the input dictionary + self.assertIsNone(metadata.content_cipher_tag_length) + self.assertIsNone(metadata.instruction_file) + + def test_to_dict(self): + # Create an ObjectMetadata instance with some fields set + metadata = ObjectMetadata( + encrypted_data_key_v2="encrypted-key-data", + encrypted_data_key_algorithm="kms+context", + content_iv="base64-encoded-iv", + content_cipher="AES/GCM/NoPadding" + ) + + # Convert to dictionary + metadata_dict = metadata.to_dict() + + # Verify that the dictionary contains the expected keys and values + self.assertEqual(metadata_dict["x-amz-key-v2"], "encrypted-key-data") + self.assertEqual(metadata_dict["x-amz-wrap-alg"], "kms+context") + self.assertEqual(metadata_dict["x-amz-iv"], "base64-encoded-iv") + self.assertEqual(metadata_dict["x-amz-cek-alg"], "AES/GCM/NoPadding") + + # Verify that fields that are None are not included in the dictionary + self.assertNotIn("x-amz-key", metadata_dict) + self.assertNotIn("x-amz-matdesc", metadata_dict) + # Note: content_cipher_tag_length has a default value of "128" + self.assertEqual(metadata_dict.get("x-amz-tag-len"), "128") + self.assertNotIn("x-amz-crypto-instr-file", metadata_dict) + + def test_roundtrip(self): + # Create a metadata dictionary + original_dict = { + "x-amz-key-v2": "encrypted-key-data", + "x-amz-wrap-alg": "kms+context", + "x-amz-iv": "base64-encoded-iv", + "x-amz-cek-alg": "AES/GCM/NoPadding" + } + + # Convert to ObjectMetadata and back to dictionary + metadata = ObjectMetadata.from_dict(original_dict) + result_dict = metadata.to_dict() + + # Remove the tag length field which has a default value + if "x-amz-tag-len" in result_dict: + result_dict.pop("x-amz-tag-len") + + # Verify that the result matches the original + self.assertEqual(result_dict, original_dict) + + +if __name__ == "__main__": + unittest.main() From 62edf8a70440a44f329c482e6bcdf639ebd20b8f Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 6 Aug 2025 15:11:30 -0700 Subject: [PATCH 03/81] include decryptionMaterials change --- .../materials/crypto_materials_manager.py | 27 ++-- src/s3_encryption/materials/keyring.py | 26 ++-- src/s3_encryption/materials/kms_keyring.py | 19 +-- src/s3_encryption/materials/materials.py | 69 +++++++++- src/s3_encryption/pipelines.py | 28 +++-- test/test_decryption_materials.py | 89 +++++++++++++ test/test_decryption_materials_integration.py | 118 ++++++++++++++++++ 7 files changed, 336 insertions(+), 40 deletions(-) create mode 100644 test/test_decryption_materials.py create mode 100644 test/test_decryption_materials_integration.py diff --git a/src/s3_encryption/materials/crypto_materials_manager.py b/src/s3_encryption/materials/crypto_materials_manager.py index 33910d75..f30f3491 100644 --- a/src/s3_encryption/materials/crypto_materials_manager.py +++ b/src/s3_encryption/materials/crypto_materials_manager.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 from attrs import define from .keyring import AbstractKeyring -from .materials import EncryptionMaterials -from typing import List, Dict, Any +from .materials import EncryptionMaterials, DecryptionMaterials +from typing import List, Dict, Any, Union # API Stub for CMM class AbstractCryptoMaterialsManager(): @@ -24,10 +24,10 @@ def decryptMaterials(self, decMatsRequest): Decrypt materials using the keyring. Args: - decMatsRequest (Dict[str, Any]): Request containing decryption parameters + decMatsRequest (Dict[str, Any] or DecryptionMaterials): Request containing decryption parameters Returns: - Dict[str, Any]: The decryption materials + DecryptionMaterials: The decryption materials """ raise NotImplementedError @@ -56,7 +56,20 @@ def getEncryptionMaterials(self, encMatsRequest): return self.keyring.onEncrypt(materials) def decryptMaterials(self, decMatsRequest): - # TODO: Fill with defaults + stuff from decMatsRequest - materials = {**decMatsRequest} - encrypted_data_keys = decMatsRequest.get('encrypted_data_keys') + """ + Decrypt materials using the keyring. + + Args: + decMatsRequest (Dict[str, Any] or DecryptionMaterials): Request containing decryption parameters + + Returns: + DecryptionMaterials: The decryption materials + """ + # Convert dictionary to DecryptionMaterials if needed + if isinstance(decMatsRequest, dict): + materials = DecryptionMaterials.from_dict(decMatsRequest) + else: + materials = decMatsRequest + + encrypted_data_keys = materials.encrypted_data_keys return self.keyring.onDecrypt(materials, encrypted_data_keys) diff --git a/src/s3_encryption/materials/keyring.py b/src/s3_encryption/materials/keyring.py index 78a884e9..222eec7b 100644 --- a/src/s3_encryption/materials/keyring.py +++ b/src/s3_encryption/materials/keyring.py @@ -3,7 +3,8 @@ from attrs import define, field from ..exceptions import S3EncryptionClientError -from .materials import EncryptionMaterials +from .materials import EncryptionMaterials, DecryptionMaterials +from typing import List, Optional @define class AbstractKeyring(): @@ -30,11 +31,11 @@ def onDecrypt(self, decMaterials, encrypted_data_keys=None): Decrypt one of the encrypted data keys and update decMaterials. Args: - decMaterials (dict): A dictionary containing decryption materials + decMaterials (DecryptionMaterials): A DecryptionMaterials instance containing decryption materials encrypted_data_keys (List[EncryptedDataKey], optional): A list of encrypted data keys to try. Returns: - dict: The updated decMaterials with the plaintext data key (PDK) + DecryptionMaterials: The updated decMaterials with the plaintext data key (PDK) """ raise NotImplementedError @@ -76,25 +77,28 @@ def onDecrypt(self, decMaterials, encrypted_data_keys=None): Validate decryption materials before decryption. Args: - decMaterials (dict): A dictionary containing decryption materials + decMaterials (DecryptionMaterials): A DecryptionMaterials instance containing decryption materials encrypted_data_keys (List[EncryptedDataKey], optional): A list of encrypted data keys to try. Returns: - dict: The validated decryption materials + DecryptionMaterials: The validated decryption materials """ # Validate decryption materials - if not isinstance(decMaterials, dict): - raise S3EncryptionClientError("Decryption materials must be a dictionary") + if not isinstance(decMaterials, DecryptionMaterials): + raise S3EncryptionClientError("Decryption materials must be a DecryptionMaterials instance") + + # Use encrypted_data_keys from parameters if provided, otherwise use from decMaterials + edks = encrypted_data_keys if encrypted_data_keys is not None else decMaterials.encrypted_data_keys # Validate encrypted_data_keys - if encrypted_data_keys is None or len(encrypted_data_keys) == 0: + if edks is None or len(edks) == 0: raise S3EncryptionClientError("No encrypted data keys provided") - # Ensure encryption contexts are dictionaries if present - if 'encryption_context_from_request' in decMaterials and not isinstance(decMaterials['encryption_context_from_request'], dict): + # Ensure encryption contexts are dictionaries + if not isinstance(decMaterials.encryption_context_from_request, dict): raise S3EncryptionClientError("Encryption context from request must be a dictionary") - if 'encryption_context_stored' in decMaterials and not isinstance(decMaterials['encryption_context_stored'], dict): + if not isinstance(decMaterials.encryption_context_stored, dict): raise S3EncryptionClientError("Stored encryption context must be a dictionary") return decMaterials diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index 69f10700..75d75a27 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -2,9 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 from .keyring import S3Keyring from .encrypted_data_key import EncryptedDataKey -from .materials import EncryptionMaterials +from .materials import EncryptionMaterials, DecryptionMaterials from ..exceptions import S3EncryptionClientError from attrs import define, field +from typing import List, Optional KMS_CONTEXT_DEFAULT_KEY = "aws:x-amz-cek-alg" KMS_V1_DEFAULT_KEY = "kms_cmk_id" @@ -55,18 +56,18 @@ def onDecrypt(self, decMaterials, encrypted_data_keys=None): Decrypt one of the encrypted data keys and update decMaterials. Args: - decMaterials (dict): A dictionary containing decryption materials + decMaterials (DecryptionMaterials): A DecryptionMaterials instance containing decryption materials encrypted_data_keys (List[EncryptedDataKey], optional): A list of encrypted data keys to try. Returns: - dict: The updated decMaterials with the plaintext data key (PDK) + DecryptionMaterials: The updated decMaterials with the plaintext data key (PDK) """ try: # Call parent class validation decMaterials = super().onDecrypt(decMaterials, encrypted_data_keys) - # Handle both single EDK (backward compatibility) and list of EDKs - edks = encrypted_data_keys + # Use encrypted_data_keys from parameters if provided, otherwise use from decMaterials + edks = encrypted_data_keys if encrypted_data_keys is not None else decMaterials.encrypted_data_keys # Try to decrypt each EDK until one succeeds # TODO: probably just enforce |EDKs| == 1 and remove loop @@ -75,8 +76,8 @@ def onDecrypt(self, decMaterials, encrypted_data_keys=None): try: edk_bytes = edk.encrypted_data_key if edk.key_provider_info == "kms+context": - encryption_context_from_request = decMaterials.get('encryption_context_from_request', {}) - encryption_context_stored = decMaterials.get('encryption_context_stored', {}) + encryption_context_from_request = decMaterials.encryption_context_from_request + encryption_context_stored = decMaterials.encryption_context_stored # Default EC MUST NOT be passed in via request if KMS_CONTEXT_DEFAULT_KEY in encryption_context_from_request: @@ -100,9 +101,9 @@ def onDecrypt(self, decMaterials, encrypted_data_keys=None): response = self.kms_client.decrypt( KeyId = self.kms_key_id, CiphertextBlob = edk_bytes, - EncryptionContext = decMaterials['encryption_context_stored'] + EncryptionContext = decMaterials.encryption_context_stored ) - decMaterials['PDK'] = response['Plaintext'] + decMaterials.plaintext_data_key = response['Plaintext'] return decMaterials except Exception as e: last_exception = e diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index 32b0b880..f532dfb5 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -1,7 +1,7 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 from attrs import define, field -from typing import Optional, Dict, Any +from typing import Optional, Dict, Any, List from .encrypted_data_key import EncryptedDataKey @define @@ -57,3 +57,70 @@ def to_dict(self) -> Dict[str, Any]: result['PDK'] = self.plaintext_data_key return result + + +@define +class DecryptionMaterials: + """ + Class representing decryption materials for S3 encryption. + + This class provides a structured way to handle decryption materials + with fields corresponding to the data needed for decryption operations. + + Attributes: + iv (Optional[bytes]): The initialization vector used for content encryption + encrypted_data_keys (List[EncryptedDataKey]): List of encrypted data keys to try + encryption_context_stored (Dict[str, str]): Encryption context stored with the object + encryption_context_from_request (Dict[str, str]): Encryption context provided in the request + plaintext_data_key (Optional[bytes]): The plaintext data key (PDK) + """ + iv: Optional[bytes] = field(default=None) + encrypted_data_keys: List[EncryptedDataKey] = field(factory=list) + encryption_context_stored: Dict[str, str] = field(factory=dict) + encryption_context_from_request: Dict[str, str] = field(factory=dict) + plaintext_data_key: Optional[bytes] = field(default=None) + + @classmethod + def from_dict(cls, materials_dict: Dict[str, Any]) -> 'DecryptionMaterials': + """ + Create a DecryptionMaterials instance from a dictionary. + + Args: + materials_dict (Dict[str, Any]): Dictionary containing decryption materials + + Returns: + DecryptionMaterials: A new instance with fields populated from the dictionary + """ + return cls( + iv=materials_dict.get('iv'), + encrypted_data_keys=materials_dict.get('encrypted_data_keys', []), + encryption_context_stored=materials_dict.get('encryption_context_stored', {}), + encryption_context_from_request=materials_dict.get('encryption_context_from_request', {}), + plaintext_data_key=materials_dict.get('PDK') + ) + + def to_dict(self) -> Dict[str, Any]: + """ + Convert the DecryptionMaterials instance to a dictionary. + + Returns: + Dict[str, Any]: Dictionary containing decryption materials + """ + result = {} + + if self.iv is not None: + result['iv'] = self.iv + + if self.encrypted_data_keys: + result['encrypted_data_keys'] = self.encrypted_data_keys + + if self.encryption_context_stored: + result['encryption_context_stored'] = self.encryption_context_stored + + if self.encryption_context_from_request: + result['encryption_context_from_request'] = self.encryption_context_from_request + + if self.plaintext_data_key is not None: + result['PDK'] = self.plaintext_data_key + + return result diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 7ddee40c..ae164be8 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -5,9 +5,10 @@ import base64 from .materials.crypto_materials_manager import AbstractCryptoMaterialsManager from .materials.encrypted_data_key import EncryptedDataKey -from .materials.materials import EncryptionMaterials +from .materials.materials import EncryptionMaterials, DecryptionMaterials from cryptography.hazmat.primitives.ciphers.aead import AESGCM from .metadata import ObjectMetadata +from typing import Dict, Any, Optional, List, Union @define class PutEncryptedObjectPipeline: @@ -92,11 +93,11 @@ def decrypt(self, response, encryption_context={}): Decrypt the data after it is retrieved from S3. Args: - encrypted_data (bytes): The encrypted data retrieved from S3 - encryption_metadata (dict, optional): Metadata about the encryption + response (dict): The response from S3 containing the encrypted data and metadata + encryption_context (dict, optional): Additional context for decryption Returns: - bytes or str: The decrypted data + bytes: The decrypted data """ # Convert the metadata dictionary to an ObjectMetadata instance encrypted_data = response.get('Body').read() @@ -131,15 +132,18 @@ def decrypt(self, response, encryption_context={}): ) encrypted_data_keys.append(legacy_encrypted_data_key) - dec_mat_req = { - "iv": iv_bytes, - "encrypted_data_keys": encrypted_data_keys, - "encryption_context_stored": metadata.encrypted_data_key_context, - "encryption_context_from_request": encryption_context - } - dec_mats = self.cmm.decryptMaterials(dec_mat_req) + # Create a DecryptionMaterials instance + dec_materials = DecryptionMaterials( + iv=iv_bytes, + encrypted_data_keys=encrypted_data_keys, + encryption_context_stored=metadata.encrypted_data_key_context or {}, + encryption_context_from_request=encryption_context + ) + + # Get decryption materials from the crypto materials manager + dec_materials = self.cmm.decryptMaterials(dec_materials) - aesgcm = AESGCM(dec_mats['PDK']) + aesgcm = AESGCM(dec_materials.plaintext_data_key) plaintext = aesgcm.decrypt( nonce=iv_bytes, diff --git a/test/test_decryption_materials.py b/test/test_decryption_materials.py new file mode 100644 index 00000000..d6c08530 --- /dev/null +++ b/test/test_decryption_materials.py @@ -0,0 +1,89 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import unittest +from src.s3_encryption.materials.materials import DecryptionMaterials +from src.s3_encryption.materials.encrypted_data_key import EncryptedDataKey + +class TestDecryptionMaterials(unittest.TestCase): + def test_create_decryption_materials(self): + """Test creating a DecryptionMaterials instance.""" + materials = DecryptionMaterials() + self.assertEqual(materials.encrypted_data_keys, []) + self.assertEqual(materials.encryption_context_stored, {}) + self.assertEqual(materials.encryption_context_from_request, {}) + self.assertIsNone(materials.iv) + self.assertIsNone(materials.plaintext_data_key) + + def test_create_with_parameters(self): + """Test creating a DecryptionMaterials instance with parameters.""" + iv = b'initialization-vector' + encrypted_data_keys = [ + EncryptedDataKey( + key_provider_id=b'S3Keyring', + key_provider_info="kms+context", + encrypted_data_key=b'encrypted-data-key' + ) + ] + encryption_context_stored = {"key1": "value1"} + encryption_context_from_request = {"key2": "value2"} + plaintext_data_key = b'plaintext-data-key' + + materials = DecryptionMaterials( + iv=iv, + encrypted_data_keys=encrypted_data_keys, + encryption_context_stored=encryption_context_stored, + encryption_context_from_request=encryption_context_from_request, + plaintext_data_key=plaintext_data_key + ) + + self.assertEqual(materials.iv, iv) + self.assertEqual(materials.encrypted_data_keys, encrypted_data_keys) + self.assertEqual(materials.encryption_context_stored, encryption_context_stored) + self.assertEqual(materials.encryption_context_from_request, encryption_context_from_request) + self.assertEqual(materials.plaintext_data_key, plaintext_data_key) + + def test_from_dict(self): + """Test creating a DecryptionMaterials instance from a dictionary.""" + edk = EncryptedDataKey( + key_provider_id=b'S3Keyring', + key_provider_info="kms+context", + encrypted_data_key=b'encrypted-data-key' + ) + materials_dict = { + 'iv': b'initialization-vector', + 'encrypted_data_keys': [edk], + 'encryption_context_stored': {"key1": "value1"}, + 'encryption_context_from_request': {"key2": "value2"}, + 'PDK': b'plaintext-data-key' + } + materials = DecryptionMaterials.from_dict(materials_dict) + self.assertEqual(materials.iv, b'initialization-vector') + self.assertEqual(materials.encrypted_data_keys, [edk]) + self.assertEqual(materials.encryption_context_stored, {"key1": "value1"}) + self.assertEqual(materials.encryption_context_from_request, {"key2": "value2"}) + self.assertEqual(materials.plaintext_data_key, b'plaintext-data-key') + + def test_to_dict(self): + """Test converting a DecryptionMaterials instance to a dictionary.""" + edk = EncryptedDataKey( + key_provider_id=b'S3Keyring', + key_provider_info="kms+context", + encrypted_data_key=b'encrypted-data-key' + ) + materials = DecryptionMaterials( + iv=b'initialization-vector', + encrypted_data_keys=[edk], + encryption_context_stored={"key1": "value1"}, + encryption_context_from_request={"key2": "value2"}, + plaintext_data_key=b'plaintext-data-key' + ) + materials_dict = materials.to_dict() + self.assertEqual(materials_dict['iv'], b'initialization-vector') + self.assertEqual(materials_dict['encrypted_data_keys'], [edk]) + self.assertEqual(materials_dict['encryption_context_stored'], {"key1": "value1"}) + self.assertEqual(materials_dict['encryption_context_from_request'], {"key2": "value2"}) + self.assertEqual(materials_dict['PDK'], b'plaintext-data-key') + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_decryption_materials_integration.py b/test/test_decryption_materials_integration.py new file mode 100644 index 00000000..5086715a --- /dev/null +++ b/test/test_decryption_materials_integration.py @@ -0,0 +1,118 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import unittest +from unittest.mock import MagicMock, patch +from src.s3_encryption.materials.materials import DecryptionMaterials +from src.s3_encryption.materials.encrypted_data_key import EncryptedDataKey +from src.s3_encryption.materials.keyring import S3Keyring +from src.s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager + +class TestDecryptionMaterialsIntegration(unittest.TestCase): + def test_keyring_onDecrypt(self): + """Test that S3Keyring.onDecrypt properly handles DecryptionMaterials.""" + # Create a keyring + keyring = S3Keyring() + + # Create an encrypted data key + edk = EncryptedDataKey( + key_provider_id=b'S3Keyring', + key_provider_info="kms+context", + encrypted_data_key=b'encrypted-data-key' + ) + + # Create decryption materials + materials = DecryptionMaterials( + iv=b'initialization-vector', + encrypted_data_keys=[edk], + encryption_context_stored={"key1": "value1"}, + encryption_context_from_request={"key2": "value2"} + ) + + # Mock the validation method to return the materials + with patch.object(S3Keyring, 'onDecrypt', return_value=materials) as mock_onDecrypt: + # Call onDecrypt + result = keyring.onDecrypt(materials, [edk]) + + # Verify the result is a DecryptionMaterials instance + self.assertIsInstance(result, DecryptionMaterials) + self.assertEqual(result.iv, b'initialization-vector') + self.assertEqual(result.encrypted_data_keys, [edk]) + self.assertEqual(result.encryption_context_stored, {"key1": "value1"}) + self.assertEqual(result.encryption_context_from_request, {"key2": "value2"}) + + def test_cmm_decryptMaterials_with_dict(self): + """Test that DefaultCryptoMaterialsManager.decryptMaterials properly handles dictionary input.""" + # Create a mock keyring + keyring = MagicMock() + edk = EncryptedDataKey( + key_provider_id=b'S3Keyring', + key_provider_info="kms+context", + encrypted_data_key=b'encrypted-data-key' + ) + keyring.onDecrypt.return_value = DecryptionMaterials( + iv=b'initialization-vector', + encrypted_data_keys=[edk], + encryption_context_stored={"key1": "value1"}, + encryption_context_from_request={"key2": "value2"}, + plaintext_data_key=b'plaintext-data-key' + ) + + # Create a CMM + cmm = DefaultCryptoMaterialsManager(keyring=keyring) + + # Call decryptMaterials with a dictionary + result = cmm.decryptMaterials({ + 'iv': b'initialization-vector', + 'encrypted_data_keys': [edk], + 'encryption_context_stored': {"key1": "value1"}, + 'encryption_context_from_request': {"key2": "value2"} + }) + + # Verify the result is a DecryptionMaterials instance + self.assertIsInstance(result, DecryptionMaterials) + self.assertEqual(result.iv, b'initialization-vector') + self.assertEqual(result.encrypted_data_keys, [edk]) + self.assertEqual(result.encryption_context_stored, {"key1": "value1"}) + self.assertEqual(result.encryption_context_from_request, {"key2": "value2"}) + self.assertEqual(result.plaintext_data_key, b'plaintext-data-key') + + def test_cmm_decryptMaterials_with_materials(self): + """Test that DefaultCryptoMaterialsManager.decryptMaterials properly handles DecryptionMaterials input.""" + # Create a mock keyring + keyring = MagicMock() + edk = EncryptedDataKey( + key_provider_id=b'S3Keyring', + key_provider_info="kms+context", + encrypted_data_key=b'encrypted-data-key' + ) + keyring.onDecrypt.return_value = DecryptionMaterials( + iv=b'initialization-vector', + encrypted_data_keys=[edk], + encryption_context_stored={"key1": "value1"}, + encryption_context_from_request={"key2": "value2"}, + plaintext_data_key=b'plaintext-data-key' + ) + + # Create a CMM + cmm = DefaultCryptoMaterialsManager(keyring=keyring) + + # Call decryptMaterials with a DecryptionMaterials instance + materials = DecryptionMaterials( + iv=b'initialization-vector', + encrypted_data_keys=[edk], + encryption_context_stored={"key1": "value1"}, + encryption_context_from_request={"key2": "value2"} + ) + result = cmm.decryptMaterials(materials) + + # Verify the result is a DecryptionMaterials instance + self.assertIsInstance(result, DecryptionMaterials) + self.assertEqual(result.iv, b'initialization-vector') + self.assertEqual(result.encrypted_data_keys, [edk]) + self.assertEqual(result.encryption_context_stored, {"key1": "value1"}) + self.assertEqual(result.encryption_context_from_request, {"key2": "value2"}) + self.assertEqual(result.plaintext_data_key, b'plaintext-data-key') + +if __name__ == '__main__': + unittest.main() From d4baef14ce81be3a07f37472854b989d9dad677f Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 6 Aug 2025 15:25:55 -0700 Subject: [PATCH 04/81] add Github workflows --- .github/workflows/main.yml | 21 ++++++++++++++++ .github/workflows/test.yml | 49 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..b7396de4 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,21 @@ +name: Main Workflow + +on: + push: + branches: [ main ] + pull_request: + workflow_dispatch: + inputs: + python-version: + description: 'Python version to use' + default: '3.11' + required: false + type: string + +jobs: + run-tests: + name: Run Tests + uses: ./.github/workflows/test.yml + with: + python-version: ${{ inputs.python-version || '3.11' }} + secrets: inherit diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..4b5db913 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,49 @@ +name: Run Tests + +on: + workflow_call: + # Optional inputs that can be provided when calling this workflow + inputs: + python-version: + description: 'Python version to use' + default: '3.11' + required: false + type: string + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version || '3.11' }} + + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: 1.7.1 + virtualenvs-create: true + virtualenvs-in-project: true + + - name: Install dependencies + run: poetry install + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role + aws-region: us-west-2 + + - name: Run unit tests + run: poetry run pytest test/ --verbose + + - name: Run integration tests + run: poetry run pytest test/integration/ --verbose + env: + CI_S3_BUCKET: ${{ secrets.CI_S3_BUCKET }} + CI_KMS_KEY_ALIAS: ${{ secrets.CI_KMS_KEY_ALIAS }} From fae581968fce072ab63e4e9549a770569d35dab5 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 6 Aug 2025 15:35:30 -0700 Subject: [PATCH 05/81] permissions --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4b5db913..8cb91e38 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,6 +13,9 @@ on: jobs: test: runs-on: ubuntu-latest + permissions: + id-token: write + contents: read steps: - name: Checkout code From 99c9591f241cf4fd3b83f042472693218e9b557f Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 6 Aug 2025 15:42:19 -0700 Subject: [PATCH 06/81] fix test --- test/integration/test_i_s3_encryption.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index 981fbd3c..db96aa06 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -1,11 +1,13 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 import boto3 +import os from datetime import datetime from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.materials.kms_keyring import KmsKeyring -bucket = "s3-ec-python-v3-test" +bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") +kms_key_id = os.environ.get("CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key") def test_simple_roundtrip(): key = "simple-rt" @@ -13,7 +15,6 @@ def test_simple_roundtrip(): data = "test input for simple v3 round trip" - kms_key_id = "arn:aws:kms:us-east-2:657301468084:key/1f469b1a-5cfa-4879-9bdf-27b3abd9b8d5" kms_client = boto3.client("kms", region_name="us-east-2") keyring = KmsKeyring(kms_client, kms_key_id) @@ -38,4 +39,4 @@ def test_simple_roundtrip(): print("Output:") print(output) else: - print("Success!") \ No newline at end of file + print("Success!") From 52dc9ab7dd7986c75abf971f63494708fd0d5cd7 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 6 Aug 2025 15:54:11 -0700 Subject: [PATCH 07/81] fix region --- test/integration/test_i_s3_encryption.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index db96aa06..b70e9c09 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -7,6 +7,7 @@ from s3_encryption.materials.kms_keyring import KmsKeyring bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") +region = os.environ.get("CI_AWS_REGION", "us-west-2") kms_key_id = os.environ.get("CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key") def test_simple_roundtrip(): @@ -15,7 +16,7 @@ def test_simple_roundtrip(): data = "test input for simple v3 round trip" - kms_client = boto3.client("kms", region_name="us-east-2") + kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) From 61096c67f1cb53c70e39a3a7f8946d56ae12b384 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 6 Aug 2025 16:02:39 -0700 Subject: [PATCH 08/81] debug --- src/s3_encryption/materials/keyring.py | 3 ++ test/test_decryption_materials_integration.py | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/s3_encryption/materials/keyring.py b/src/s3_encryption/materials/keyring.py index 222eec7b..e4c6d338 100644 --- a/src/s3_encryption/materials/keyring.py +++ b/src/s3_encryption/materials/keyring.py @@ -96,6 +96,9 @@ def onDecrypt(self, decMaterials, encrypted_data_keys=None): # Ensure encryption contexts are dictionaries if not isinstance(decMaterials.encryption_context_from_request, dict): + print("EC from req: ") + print(decMaterials.encryption_context_from_request) + print("now raising..") raise S3EncryptionClientError("Encryption context from request must be a dictionary") if not isinstance(decMaterials.encryption_context_stored, dict): diff --git a/test/test_decryption_materials_integration.py b/test/test_decryption_materials_integration.py index 5086715a..dbbaf431 100644 --- a/test/test_decryption_materials_integration.py +++ b/test/test_decryption_materials_integration.py @@ -41,6 +41,38 @@ def test_keyring_onDecrypt(self): self.assertEqual(result.encryption_context_stored, {"key1": "value1"}) self.assertEqual(result.encryption_context_from_request, {"key2": "value2"}) + def test_keyring_onDecrypt_default_EC(self): + """Test that S3Keyring.onDecrypt properly handles DecryptionMaterials.""" + # Create a keyring + keyring = S3Keyring() + + # Create an encrypted data key + edk = EncryptedDataKey( + key_provider_id=b'S3Keyring', + key_provider_info="kms+context", + encrypted_data_key=b'encrypted-data-key' + ) + + # Create decryption materials + materials = DecryptionMaterials( + iv=b'initialization-vector', + encrypted_data_keys=[edk], + encryption_context_stored={}, + encryption_context_from_request={} + ) + + # Mock the validation method to return the materials + with patch.object(S3Keyring, 'onDecrypt', return_value=materials) as mock_onDecrypt: + # Call onDecrypt + result = keyring.onDecrypt(materials, [edk]) + + # Verify the result is a DecryptionMaterials instance + self.assertIsInstance(result, DecryptionMaterials) + self.assertEqual(result.iv, b'initialization-vector') + self.assertEqual(result.encrypted_data_keys, [edk]) + self.assertEqual(result.encryption_context_stored, {}) + self.assertEqual(result.encryption_context_from_request, {}) + def test_cmm_decryptMaterials_with_dict(self): """Test that DefaultCryptoMaterialsManager.decryptMaterials properly handles dictionary input.""" # Create a mock keyring From 23362e3d9434d30e74de3bf943fdfce754b567d4 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 6 Aug 2025 16:10:12 -0700 Subject: [PATCH 09/81] or dict --- src/s3_encryption/pipelines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index ae164be8..41ec53dc 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -137,7 +137,7 @@ def decrypt(self, response, encryption_context={}): iv=iv_bytes, encrypted_data_keys=encrypted_data_keys, encryption_context_stored=metadata.encrypted_data_key_context or {}, - encryption_context_from_request=encryption_context + encryption_context_from_request=encryption_context or {} ) # Get decryption materials from the crypto materials manager From 8c76a5bb52167a85f74d9df7fbfe92f47f0aab38 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 6 Aug 2025 16:24:06 -0700 Subject: [PATCH 10/81] debug --- test/integration/test_i_s3_encryption.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index b70e9c09..29b96301 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -18,6 +18,7 @@ def test_simple_roundtrip(): kms_client = boto3.client("kms", region_name=region) + print("KMS Key: " + kms_key_id) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") From 24bcb4899a259a4acaf2c50f5ed64c8820947696 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 6 Aug 2025 16:27:15 -0700 Subject: [PATCH 11/81] github env vars --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b7396de4..102b8a74 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,3 +19,6 @@ jobs: with: python-version: ${{ inputs.python-version || '3.11' }} secrets: inherit + env: + CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} + CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} From ef416de7458291fe5bfefd6131464902827c50ff Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 6 Aug 2025 16:29:33 -0700 Subject: [PATCH 12/81] vars not secrets --- .github/workflows/main.yml | 3 --- .github/workflows/test.yml | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 102b8a74..b7396de4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,6 +19,3 @@ jobs: with: python-version: ${{ inputs.python-version || '3.11' }} secrets: inherit - env: - CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} - CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8cb91e38..c60d4a20 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -48,5 +48,5 @@ jobs: - name: Run integration tests run: poetry run pytest test/integration/ --verbose env: - CI_S3_BUCKET: ${{ secrets.CI_S3_BUCKET }} - CI_KMS_KEY_ALIAS: ${{ secrets.CI_KMS_KEY_ALIAS }} + CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} + CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} From 480a3987080e3b8c0e9dcfa61e1098134c6bb388 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 6 Aug 2025 16:31:42 -0700 Subject: [PATCH 13/81] remove debug, raise error on mismatch --- src/s3_encryption/materials/keyring.py | 3 --- test/integration/test_i_s3_encryption.py | 5 +---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/s3_encryption/materials/keyring.py b/src/s3_encryption/materials/keyring.py index e4c6d338..222eec7b 100644 --- a/src/s3_encryption/materials/keyring.py +++ b/src/s3_encryption/materials/keyring.py @@ -96,9 +96,6 @@ def onDecrypt(self, decMaterials, encrypted_data_keys=None): # Ensure encryption contexts are dictionaries if not isinstance(decMaterials.encryption_context_from_request, dict): - print("EC from req: ") - print(decMaterials.encryption_context_from_request) - print("now raising..") raise S3EncryptionClientError("Encryption context from request must be a dictionary") if not isinstance(decMaterials.encryption_context_stored, dict): diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index 29b96301..55b1d678 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -18,27 +18,24 @@ def test_simple_roundtrip(): kms_client = boto3.client("kms", region_name=region) - print("KMS Key: " + kms_key_id) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) s3ec = S3EncryptionClient(wrapped_client, config) s3ec.put_object(Bucket=bucket, Key=key, Data=data) - print("put object success!") get_req = { 'Bucket': bucket, 'Key': key } response = s3ec.get_object(**get_req) output = response['Body'].read().decode('utf-8') - print("get succeeded!") - print(response) if output != data: print("Uh oh! Input and output don't match!") print("Input:") print(input) print("Output:") print(output) + raise RuntimeError else: print("Success!") From 21b59063e7e837e9fd7f423e34b110fbaebe595f Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 6 Aug 2025 17:16:31 -0700 Subject: [PATCH 14/81] run formatter, add linting --- Makefile | 38 ++++ README.md | 73 +++++++ poetry.lock | 204 +++++++++++++++++- pyproject.toml | 23 ++ src/s3_encryption/__init__.py | 67 +++--- src/s3_encryption/materials/__init__.py | 16 +- .../materials/crypto_materials_manager.py | 36 ++-- .../materials/encrypted_data_key.py | 6 +- src/s3_encryption/materials/keyring.py | 62 +++--- src/s3_encryption/materials/kms_keyring.py | 73 ++++--- src/s3_encryption/materials/materials.py | 93 ++++---- src/s3_encryption/metadata.py | 38 ++-- src/s3_encryption/pipelines.py | 97 ++++----- test/integration/test_i_s3_encryption.py | 18 +- test/test_decryption_materials.py | 63 +++--- test/test_decryption_materials_integration.py | 109 +++++----- test/test_encryption_materials.py | 37 ++-- test/test_encryption_materials_integration.py | 51 +++-- test/test_metadata.py | 34 +-- 19 files changed, 763 insertions(+), 375 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..15dd281a --- /dev/null +++ b/Makefile @@ -0,0 +1,38 @@ +.PHONY: lint format test test-unit test-integration install + +# Default target +all: lint test + +# Install dependencies +install: + poetry install + +# Run linting checks +lint: + poetry run black --check . + poetry run isort --check . + # Allow flake8 to fail for now as we're gradually adopting linting standards + poetry run flake8 src/ test/ || true + +# Format code with Black and isort +format: + poetry run black . + poetry run isort . + +# Run all tests +test: test-unit test-integration + +# Run unit tests +test-unit: + poetry run pytest test/ --verbose + +# Run integration tests +test-integration: + poetry run pytest test/integration/ --verbose + +# Clean up cache files +clean: + find . -type d -name __pycache__ -exec rm -rf {} + + find . -type d -name .pytest_cache -exec rm -rf {} + + find . -type d -name .coverage -exec rm -rf {} + + find . -type f -name "*.pyc" -delete diff --git a/README.md b/README.md index 2104b640..c24d1796 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,75 @@ # Amazon S3 Encryption Client Python +This library provides an S3 client that supports client-side encryption. + +## Development + +### Prerequisites + +- Python 3.11 or higher +- [Poetry](https://python-poetry.org/) for dependency management + +### Setup + +Install dependencies: + +```bash +make install +``` + +### Testing + +Run all tests: + +```bash +make test +``` + +Run unit tests only: + +```bash +make test-unit +``` + +Run integration tests only: + +```bash +make test-integration +``` + +### Code Quality + +This project uses [Black](https://black.readthedocs.io/) for code formatting, [isort](https://pycqa.github.io/isort/) for import sorting, and [Flake8](https://flake8.pycqa.org/) for linting. + +Check code quality: + +```bash +make lint +``` + +Format code with Black and isort: + +```bash +make format +``` + +Clean up cache files: + +```bash +make clean +``` + +#### Linting Standards + +The project is configured with Black, isort, and Flake8 to enforce consistent code style and quality. Currently, Flake8 is set to report issues but not fail the build, allowing for gradual adoption of linting standards. + +Common Flake8 issues in the codebase include: + +- **Missing docstrings** (D100-D104): Add docstrings to modules, classes, and functions +- **Docstring formatting** (D200, D212, D415): Follow Google docstring style +- **Line length** (E501): Keep lines under 100 characters +- **Unused imports** (F401): Remove unused imports +- **Unused variables** (F841): Remove or use assigned variables +- **Code complexity** (C901): Refactor complex functions + +When contributing to this project, please try to fix linting issues in the files you modify. diff --git a/poetry.lock b/poetry.lock index 1ab72864..a89805ee 100644 --- a/poetry.lock +++ b/poetry.lock @@ -96,6 +96,50 @@ files = [ DafnyRuntimePython = "4.9.0" pytz = ">=2023.3.post1,<2025.0.0" +[[package]] +name = "black" +version = "24.10.0" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.9" +files = [ + {file = "black-24.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6668650ea4b685440857138e5fe40cde4d652633b1bdffc62933d0db4ed9812"}, + {file = "black-24.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1c536fcf674217e87b8cc3657b81809d3c085d7bf3ef262ead700da345bfa6ea"}, + {file = "black-24.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:649fff99a20bd06c6f727d2a27f401331dc0cc861fb69cde910fe95b01b5928f"}, + {file = "black-24.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:fe4d6476887de70546212c99ac9bd803d90b42fc4767f058a0baa895013fbb3e"}, + {file = "black-24.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5a2221696a8224e335c28816a9d331a6c2ae15a2ee34ec857dcf3e45dbfa99ad"}, + {file = "black-24.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9da3333530dbcecc1be13e69c250ed8dfa67f43c4005fb537bb426e19200d50"}, + {file = "black-24.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4007b1393d902b48b36958a216c20c4482f601569d19ed1df294a496eb366392"}, + {file = "black-24.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:394d4ddc64782e51153eadcaaca95144ac4c35e27ef9b0a42e121ae7e57a9175"}, + {file = "black-24.10.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b5e39e0fae001df40f95bd8cc36b9165c5e2ea88900167bddf258bacef9bbdc3"}, + {file = "black-24.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d37d422772111794b26757c5b55a3eade028aa3fde43121ab7b673d050949d65"}, + {file = "black-24.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14b3502784f09ce2443830e3133dacf2c0110d45191ed470ecb04d0f5f6fcb0f"}, + {file = "black-24.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:30d2c30dc5139211dda799758559d1b049f7f14c580c409d6ad925b74a4208a8"}, + {file = "black-24.10.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cbacacb19e922a1d75ef2b6ccaefcd6e93a2c05ede32f06a21386a04cedb981"}, + {file = "black-24.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1f93102e0c5bb3907451063e08b9876dbeac810e7da5a8bfb7aeb5a9ef89066b"}, + {file = "black-24.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ddacb691cdcdf77b96f549cf9591701d8db36b2f19519373d60d31746068dbf2"}, + {file = "black-24.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:680359d932801c76d2e9c9068d05c6b107f2584b2a5b88831c83962eb9984c1b"}, + {file = "black-24.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:17374989640fbca88b6a448129cd1745c5eb8d9547b464f281b251dd00155ccd"}, + {file = "black-24.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:63f626344343083322233f175aaf372d326de8436f5928c042639a4afbbf1d3f"}, + {file = "black-24.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfa1d0cb6200857f1923b602f978386a3a2758a65b52e0950299ea014be6800"}, + {file = "black-24.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:2cd9c95431d94adc56600710f8813ee27eea544dd118d45896bb734e9d7a0dc7"}, + {file = "black-24.10.0-py3-none-any.whl", hash = "sha256:3bb2b7a1f7b685f85b11fed1ef10f8a9148bceb49853e47a294a3dd963c1dd7d"}, + {file = "black-24.10.0.tar.gz", hash = "sha256:846ea64c97afe3bc677b761787993be4991810ecc7a4a937816dd6bddedc4875"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.10)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + [[package]] name = "boto3" version = "1.39.14" @@ -213,6 +257,20 @@ files = [ [package.dependencies] pycparser = "*" +[[package]] +name = "click" +version = "8.2.1" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.10" +files = [ + {file = "click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b"}, + {file = "click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + [[package]] name = "colorama" version = "0.4.6" @@ -284,6 +342,37 @@ files = [ {file = "dafnyruntimepython-4.9.0.tar.gz", hash = "sha256:03a4c2dbbe45c13dc2c7dbefad01812367b3bb217a14b4b848d7e94ef5c08cee"}, ] +[[package]] +name = "flake8" +version = "7.3.0" +description = "the modular source code checker: pep8 pyflakes and co" +optional = false +python-versions = ">=3.9" +files = [ + {file = "flake8-7.3.0-py2.py3-none-any.whl", hash = "sha256:b9696257b9ce8beb888cdbe31cf885c90d31928fe202be0889a7cdafad32f01e"}, + {file = "flake8-7.3.0.tar.gz", hash = "sha256:fe044858146b9fc69b551a4b490d69cf960fcb78ad1edcb84e7fbb1b4a8e3872"}, +] + +[package.dependencies] +mccabe = ">=0.7.0,<0.8.0" +pycodestyle = ">=2.14.0,<2.15.0" +pyflakes = ">=3.4.0,<3.5.0" + +[[package]] +name = "flake8-docstrings" +version = "1.7.0" +description = "Extension for flake8 which uses pydocstyle to check docstrings" +optional = false +python-versions = ">=3.7" +files = [ + {file = "flake8_docstrings-1.7.0-py2.py3-none-any.whl", hash = "sha256:51f2344026da083fc084166a9353f5082b01f72901df422f74b4d953ae88ac75"}, + {file = "flake8_docstrings-1.7.0.tar.gz", hash = "sha256:4c8cc748dc16e6869728699e5d0d685da9a10b0ea718e090b1ba088e67a941af"}, +] + +[package.dependencies] +flake8 = ">=3" +pydocstyle = ">=2.1" + [[package]] name = "iniconfig" version = "2.1.0" @@ -295,6 +384,20 @@ files = [ {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, ] +[[package]] +name = "isort" +version = "5.13.2" +description = "A Python utility / library to sort Python imports." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, + {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, +] + +[package.extras] +colors = ["colorama (>=0.4.6)"] + [[package]] name = "jmespath" version = "1.0.1" @@ -306,6 +409,28 @@ files = [ {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] +[[package]] +name = "mccabe" +version = "0.7.0" +description = "McCabe checker, plugin for flake8" +optional = false +python-versions = ">=3.6" +files = [ + {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, + {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505"}, + {file = "mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558"}, +] + [[package]] name = "packaging" version = "25.0" @@ -317,6 +442,33 @@ files = [ {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, ] +[[package]] +name = "pathspec" +version = "0.12.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + +[[package]] +name = "platformdirs" +version = "4.3.8" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +optional = false +python-versions = ">=3.9" +files = [ + {file = "platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4"}, + {file = "platformdirs-4.3.8.tar.gz", hash = "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc"}, +] + +[package.extras] +docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.1.3)", "sphinx-autodoc-typehints (>=3)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.4)", "pytest-cov (>=6)", "pytest-mock (>=3.14)"] +type = ["mypy (>=1.14.1)"] + [[package]] name = "pluggy" version = "1.6.0" @@ -332,6 +484,17 @@ files = [ dev = ["pre-commit", "tox"] testing = ["coverage", "pytest", "pytest-benchmark"] +[[package]] +name = "pycodestyle" +version = "2.14.0" +description = "Python style guide checker" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pycodestyle-2.14.0-py2.py3-none-any.whl", hash = "sha256:dd6bf7cb4ee77f8e016f9c8e74a35ddd9f67e1d5fd4184d86c3b98e07099f42d"}, + {file = "pycodestyle-2.14.0.tar.gz", hash = "sha256:c4b5b517d278089ff9d0abdec919cd97262a3367449ea1c8b49b91529167b783"}, +] + [[package]] name = "pycparser" version = "2.22" @@ -343,6 +506,34 @@ files = [ {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, ] +[[package]] +name = "pydocstyle" +version = "6.3.0" +description = "Python docstring style checker" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pydocstyle-6.3.0-py3-none-any.whl", hash = "sha256:118762d452a49d6b05e194ef344a55822987a462831ade91ec5c06fd2169d019"}, + {file = "pydocstyle-6.3.0.tar.gz", hash = "sha256:7ce43f0c0ac87b07494eb9c0b462c0b73e6ff276807f204d6b53edc72b7e44e1"}, +] + +[package.dependencies] +snowballstemmer = ">=2.2.0" + +[package.extras] +toml = ["tomli (>=1.2.3)"] + +[[package]] +name = "pyflakes" +version = "3.4.0" +description = "passive checker of Python programs" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pyflakes-3.4.0-py2.py3-none-any.whl", hash = "sha256:f742a7dbd0d9cb9ea41e9a24a918996e8170c799fa528688d40dd582c8265f4f"}, + {file = "pyflakes-3.4.0.tar.gz", hash = "sha256:b24f96fafb7d2ab0ec5075b7350b3d2d2218eab42003821c06344973d3ea2f58"}, +] + [[package]] name = "pygments" version = "2.19.2" @@ -431,6 +622,17 @@ files = [ {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, ] +[[package]] +name = "snowballstemmer" +version = "3.0.1" +description = "This package provides 32 stemmers for 30 languages generated from Snowball algorithms." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*" +files = [ + {file = "snowballstemmer-3.0.1-py3-none-any.whl", hash = "sha256:6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064"}, + {file = "snowballstemmer-3.0.1.tar.gz", hash = "sha256:6d5eeeec8e9f84d4d56b847692bacf79bc2c8e90c7f80ca4444ff8b6f2e52895"}, +] + [[package]] name = "urllib3" version = "2.5.0" @@ -451,4 +653,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "b1781f15e07cc26d093bb8ed243f85d126ac76a46954cc1d1ff29261f1db380c" +content-hash = "e0d80bd0119ad8c72dfd80afa69019310ce4c75a9f81e6da9bb80666657840e2" diff --git a/pyproject.toml b/pyproject.toml index fba01461..89a86e17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,9 +14,32 @@ boto3 = "^1.37.2" cryptography = "^43.0.1" aws-cryptographic-material-providers = "^1.7.4" attrs = "^25.1.0" + +[tool.poetry.group.dev.dependencies] pytest = "^8.4.1" +black = "^24.3.0" +flake8 = "^7.0.0" +flake8-docstrings = "^1.7.0" +isort = "^5.13.2" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + +[tool.black] +line-length = 100 +target-version = ["py311"] +include = '\.pyi?$' + +[tool.flake8] +max-line-length = 100 +exclude = [".git", "__pycache__", "build", "dist"] +max-complexity = 10 +ignore = ["E203", "W503"] # E203 and W503 conflict with Black +docstring-convention = "google" + +[tool.isort] +profile = "black" +line_length = 100 +known_first_party = ["s3_encryption"] diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 99493f07..ab0a77fe 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -1,84 +1,85 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 import io -from botocore.response import StreamingBody + from attrs import define, field -from .pipelines import PutEncryptedObjectPipeline, GetEncryptedObjectPipeline +from botocore.response import StreamingBody + +from .materials.crypto_materials_manager import ( + AbstractCryptoMaterialsManager, + DefaultCryptoMaterialsManager, +) from .materials.keyring import AbstractKeyring -from .materials.crypto_materials_manager import AbstractCryptoMaterialsManager, DefaultCryptoMaterialsManager from .metadata import ObjectMetadata +from .pipelines import GetEncryptedObjectPipeline, PutEncryptedObjectPipeline @define -class S3EncryptionClientConfig(): +class S3EncryptionClientConfig: """ Configuration object for the S3 Encryption Client """ + keyring: AbstractKeyring cmm: AbstractCryptoMaterialsManager = field() + @cmm.default def _default_cmm_for_keyring(self): return DefaultCryptoMaterialsManager(self.keyring) @define -class S3EncryptionClient(): +class S3EncryptionClient: wrapped_s3_client = field() config: S3EncryptionClientConfig = field() - # TODO: I don't know exactly how boto3 works, - # we maybe instead prefer only using kwargs? - # Do we need to provide specific arg overloads? # TODO: rename Data-> Body to match boto def put_object(self, Bucket, Key, Data, EncryptionContext=None, **kwargs): # Create a pipeline for this operation pipeline = PutEncryptedObjectPipeline(self.config.cmm) - # Encrypt the data using the pipeline data_bytes = Data # We probably just shouldn't support strings, use utf8 for now + # TODO: look deeper into this, what does normal boto3 do? if type(Data) == str: - data_bytes = Data.encode('utf-8') - encrypted_data, encryption_metadata = pipeline.encrypt(data_bytes, encryption_context=EncryptionContext) - + data_bytes = Data.encode("utf-8") + encrypted_data, encryption_metadata = pipeline.encrypt( + data_bytes, encryption_context=EncryptionContext + ) + # Add encryption metadata to the request parameters - params = { - 'Bucket': Bucket, - 'Key': Key, - 'Body': encrypted_data, - **kwargs - } - + params = {"Bucket": Bucket, "Key": Key, "Body": encrypted_data, **kwargs} + # Add encryption metadata to the parameters if encryption_metadata: # Merge any existing metadata with our encryption metadata - metadata = params.get('Metadata', {}) + metadata = params.get("Metadata", {}) metadata.update(encryption_metadata) - params['Metadata'] = metadata - + params["Metadata"] = metadata + return self.wrapped_s3_client.put_object(**params) def get_object(self, EncryptionContext=None, **kwargs): # try just straight kwargs - params = { - **kwargs - } - + params = {**kwargs} + # Get the encrypted object from S3 response = self.wrapped_s3_client.get_object(**params) - + # Create a pipeline for this operation pipeline = GetEncryptedObjectPipeline(self.config.cmm) - + # Decrypt the data using the pipeline - decrypted_data = pipeline.decrypt(response, EncryptionContext) #encrypted_data, encryption_metadata) - + decrypted_data = pipeline.decrypt( + response, EncryptionContext + ) # encrypted_data, encryption_metadata) + # Create a new streaming body with the decrypted data stream = io.BytesIO(decrypted_data) streaming_body = StreamingBody(stream, len(decrypted_data)) - + # Update the response with the decrypted data - response['Body'] = streaming_body - + response["Body"] = streaming_body + return response diff --git a/src/s3_encryption/materials/__init__.py b/src/s3_encryption/materials/__init__.py index 79178052..b602bc91 100644 --- a/src/s3_encryption/materials/__init__.py +++ b/src/s3_encryption/materials/__init__.py @@ -1,17 +1,17 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from .keyring import AbstractKeyring -from .kms_keyring import KmsKeyring from .crypto_materials_manager import AbstractCryptoMaterialsManager, DefaultCryptoMaterialsManager from .encrypted_data_key import EncryptedDataKey +from .keyring import AbstractKeyring +from .kms_keyring import KmsKeyring from .materials import EncryptionMaterials __all__ = [ - 'AbstractKeyring', - 'KmsKeyring', - 'AbstractCryptoMaterialsManager', - 'DefaultCryptoMaterialsManager', - 'EncryptedDataKey', - 'EncryptionMaterials' + "AbstractKeyring", + "KmsKeyring", + "AbstractCryptoMaterialsManager", + "DefaultCryptoMaterialsManager", + "EncryptedDataKey", + "EncryptionMaterials", ] diff --git a/src/s3_encryption/materials/crypto_materials_manager.py b/src/s3_encryption/materials/crypto_materials_manager.py index f30f3491..c2d67edf 100644 --- a/src/s3_encryption/materials/crypto_materials_manager.py +++ b/src/s3_encryption/materials/crypto_materials_manager.py @@ -1,36 +1,40 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +from typing import Any, Dict, List, Union + from attrs import define + from .keyring import AbstractKeyring -from .materials import EncryptionMaterials, DecryptionMaterials -from typing import List, Dict, Any, Union +from .materials import DecryptionMaterials, EncryptionMaterials + # API Stub for CMM -class AbstractCryptoMaterialsManager(): +class AbstractCryptoMaterialsManager: def getEncryptionMaterials(self, encMatsRequest): """ Get encryption materials from the keyring. - + Args: encMatsRequest (Dict[str, Any] or EncryptionMaterials): Request containing encryption parameters - + Returns: EncryptionMaterials: The encryption materials """ raise NotImplementedError - + def decryptMaterials(self, decMatsRequest): """ Decrypt materials using the keyring. - + Args: decMatsRequest (Dict[str, Any] or DecryptionMaterials): Request containing decryption parameters - + Returns: DecryptionMaterials: The decryption materials """ raise NotImplementedError + @define class DefaultCryptoMaterialsManager(AbstractCryptoMaterialsManager): keyring: AbstractKeyring @@ -38,30 +42,30 @@ class DefaultCryptoMaterialsManager(AbstractCryptoMaterialsManager): def getEncryptionMaterials(self, encMatsRequest): """ Get encryption materials from the keyring. - + Args: encMatsRequest (Dict[str, Any]): Request containing encryption parameters - + Returns: EncryptionMaterials: The encryption materials """ # Convert dictionary to EncryptionMaterials if needed if isinstance(encMatsRequest, dict): materials = EncryptionMaterials( - encryption_context=encMatsRequest.get('encryption_context', {}) + encryption_context=encMatsRequest.get("encryption_context", {}) ) else: materials = encMatsRequest - + return self.keyring.onEncrypt(materials) - + def decryptMaterials(self, decMatsRequest): """ Decrypt materials using the keyring. - + Args: decMatsRequest (Dict[str, Any] or DecryptionMaterials): Request containing decryption parameters - + Returns: DecryptionMaterials: The decryption materials """ @@ -70,6 +74,6 @@ def decryptMaterials(self, decMatsRequest): materials = DecryptionMaterials.from_dict(decMatsRequest) else: materials = decMatsRequest - + encrypted_data_keys = materials.encrypted_data_keys return self.keyring.onDecrypt(materials, encrypted_data_keys) diff --git a/src/s3_encryption/materials/encrypted_data_key.py b/src/s3_encryption/materials/encrypted_data_key.py index 72fbeae2..0dbfa08f 100644 --- a/src/s3_encryption/materials/encrypted_data_key.py +++ b/src/s3_encryption/materials/encrypted_data_key.py @@ -2,19 +2,21 @@ # SPDX-License-Identifier: Apache-2.0 from attrs import define, field + @define class EncryptedDataKey: """ Class representing an encrypted data key. - + An encrypted data key contains information about the key provider and the encrypted data key itself. - + Attributes: key_provider_info (str): Information about the key provider key_provider_id (bytes): Identifier for the key provider encrypted_data_key (bytes): The encrypted data key """ + key_provider_info: str = field() key_provider_id: bytes = field() encrypted_data_key: bytes = field() diff --git a/src/s3_encryption/materials/keyring.py b/src/s3_encryption/materials/keyring.py index 222eec7b..98e4f6ab 100644 --- a/src/s3_encryption/materials/keyring.py +++ b/src/s3_encryption/materials/keyring.py @@ -1,26 +1,29 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +from typing import List, Optional + from attrs import define, field + from ..exceptions import S3EncryptionClientError -from .materials import EncryptionMaterials, DecryptionMaterials -from typing import List, Optional +from .materials import DecryptionMaterials, EncryptionMaterials + @define -class AbstractKeyring(): +class AbstractKeyring: # Ideally, all keyrings would inherit this field. - # However, attrs doesn't allow us to set a default here, + # However, attrs doesn't allow us to set a default here, # when inheriting keyrings have optional fields. # Even without a default it doesn't seem to play nice with attrs. - #enableLegacyWrappingAlgorithms: bool = field(default=False) + # enableLegacyWrappingAlgorithms: bool = field(default=False) def onEncrypt(self, encMaterials): """ Process encryption materials. - + Args: encMaterials (EncryptionMaterials): Encryption materials to process - + Returns: EncryptionMaterials: The processed encryption materials """ @@ -29,11 +32,11 @@ def onEncrypt(self, encMaterials): def onDecrypt(self, decMaterials, encrypted_data_keys=None): """ Decrypt one of the encrypted data keys and update decMaterials. - + Args: decMaterials (DecryptionMaterials): A DecryptionMaterials instance containing decryption materials encrypted_data_keys (List[EncryptedDataKey], optional): A list of encrypted data keys to try. - + Returns: DecryptionMaterials: The updated decMaterials with the plaintext data key (PDK) """ @@ -45,60 +48,69 @@ class S3Keyring(AbstractKeyring): """ Base class for S3 encryption keyrings that provides common validation logic. """ - # Ideally this would be set, but attrs doesn't play nice + + # Ideally this would be set, but attrs doesn't play nice # enable_legacy_wrapping_algorithms: bool = field(default=False) def onEncrypt(self, encMaterials): """ Validate encryption materials before encryption. - + Args: encMaterials (EncryptionMaterials or dict): Encryption materials - + Returns: EncryptionMaterials: The validated encryption materials """ # Convert dict to EncryptionMaterials if needed if isinstance(encMaterials, dict): encMaterials = EncryptionMaterials.from_dict(encMaterials) - + # Validate encryption materials if not isinstance(encMaterials, EncryptionMaterials): - raise S3EncryptionClientError("Encryption materials must be an EncryptionMaterials instance or a dictionary") - + raise S3EncryptionClientError( + "Encryption materials must be an EncryptionMaterials instance or a dictionary" + ) + # Ensure encryption_context is a dictionary if not isinstance(encMaterials.encryption_context, dict): raise S3EncryptionClientError("Encryption context must be a dictionary") - + return encMaterials def onDecrypt(self, decMaterials, encrypted_data_keys=None): """ Validate decryption materials before decryption. - + Args: decMaterials (DecryptionMaterials): A DecryptionMaterials instance containing decryption materials encrypted_data_keys (List[EncryptedDataKey], optional): A list of encrypted data keys to try. - + Returns: DecryptionMaterials: The validated decryption materials """ # Validate decryption materials if not isinstance(decMaterials, DecryptionMaterials): - raise S3EncryptionClientError("Decryption materials must be a DecryptionMaterials instance") - + raise S3EncryptionClientError( + "Decryption materials must be a DecryptionMaterials instance" + ) + # Use encrypted_data_keys from parameters if provided, otherwise use from decMaterials - edks = encrypted_data_keys if encrypted_data_keys is not None else decMaterials.encrypted_data_keys - + edks = ( + encrypted_data_keys + if encrypted_data_keys is not None + else decMaterials.encrypted_data_keys + ) + # Validate encrypted_data_keys if edks is None or len(edks) == 0: raise S3EncryptionClientError("No encrypted data keys provided") - + # Ensure encryption contexts are dictionaries if not isinstance(decMaterials.encryption_context_from_request, dict): raise S3EncryptionClientError("Encryption context from request must be a dictionary") - + if not isinstance(decMaterials.encryption_context_stored, dict): raise S3EncryptionClientError("Stored encryption context must be a dictionary") - + return decMaterials diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index 75d75a27..4ba5e31b 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -1,15 +1,18 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from .keyring import S3Keyring -from .encrypted_data_key import EncryptedDataKey -from .materials import EncryptionMaterials, DecryptionMaterials -from ..exceptions import S3EncryptionClientError -from attrs import define, field from typing import List, Optional +from attrs import define, field + +from ..exceptions import S3EncryptionClientError +from .encrypted_data_key import EncryptedDataKey +from .keyring import S3Keyring +from .materials import DecryptionMaterials, EncryptionMaterials + KMS_CONTEXT_DEFAULT_KEY = "aws:x-amz-cek-alg" KMS_V1_DEFAULT_KEY = "kms_cmk_id" + @define class KmsKeyring(S3Keyring): kms_client = field() @@ -19,34 +22,32 @@ class KmsKeyring(S3Keyring): def onEncrypt(self, encMaterials): """ Process encryption materials using KMS. - + Args: encMaterials (EncryptionMaterials): Encryption materials to process - + Returns: EncryptionMaterials: The processed encryption materials with KMS-generated keys """ try: # Call parent class validation encMaterials = super().onEncrypt(encMaterials) - + # Add default encryption context encryption_context = encMaterials.encryption_context encryption_context["aws:x-amz-cek-alg"] = "AES/GCM/NoPadding" response = self.kms_client.generate_data_key( - KeyId = self.kms_key_id, - KeySpec = 'AES_256', - EncryptionContext = encryption_context + KeyId=self.kms_key_id, KeySpec="AES_256", EncryptionContext=encryption_context ) # Create an EncryptedDataKey instance encrypted_data_key = EncryptedDataKey( - key_provider_id=b'S3Keyring', + key_provider_id=b"S3Keyring", key_provider_info="kms+context", - encrypted_data_key=response['CiphertextBlob'] + encrypted_data_key=response["CiphertextBlob"], ) encMaterials.encrypted_data_key = encrypted_data_key - encMaterials.plaintext_data_key = response['Plaintext'] + encMaterials.plaintext_data_key = response["Plaintext"] return encMaterials except Exception as e: raise @@ -54,21 +55,25 @@ def onEncrypt(self, encMaterials): def onDecrypt(self, decMaterials, encrypted_data_keys=None): """ Decrypt one of the encrypted data keys and update decMaterials. - + Args: decMaterials (DecryptionMaterials): A DecryptionMaterials instance containing decryption materials encrypted_data_keys (List[EncryptedDataKey], optional): A list of encrypted data keys to try. - + Returns: DecryptionMaterials: The updated decMaterials with the plaintext data key (PDK) """ try: # Call parent class validation decMaterials = super().onDecrypt(decMaterials, encrypted_data_keys) - + # Use encrypted_data_keys from parameters if provided, otherwise use from decMaterials - edks = encrypted_data_keys if encrypted_data_keys is not None else decMaterials.encrypted_data_keys - + edks = ( + encrypted_data_keys + if encrypted_data_keys is not None + else decMaterials.encrypted_data_keys + ) + # Try to decrypt each EDK until one succeeds # TODO: probably just enforce |EDKs| == 1 and remove loop last_exception = None @@ -76,12 +81,16 @@ def onDecrypt(self, decMaterials, encrypted_data_keys=None): try: edk_bytes = edk.encrypted_data_key if edk.key_provider_info == "kms+context": - encryption_context_from_request = decMaterials.encryption_context_from_request + encryption_context_from_request = ( + decMaterials.encryption_context_from_request + ) encryption_context_stored = decMaterials.encryption_context_stored # Default EC MUST NOT be passed in via request if KMS_CONTEXT_DEFAULT_KEY in encryption_context_from_request: - raise S3EncryptionClientError(f"{KMS_CONTEXT_DEFAULT_KEY} is a reserved key for the S3 encryption client") + raise S3EncryptionClientError( + f"{KMS_CONTEXT_DEFAULT_KEY} is a reserved key for the S3 encryption client" + ) # The stored EC, minus default key/values, MUST match provided EC encryption_context_stored_copy = encryption_context_stored.copy() @@ -89,26 +98,32 @@ def onDecrypt(self, decMaterials, encrypted_data_keys=None): encryption_context_stored_copy.pop(KMS_CONTEXT_DEFAULT_KEY, None) if encryption_context_stored_copy != encryption_context_from_request: # TODO: modeled error - raise S3EncryptionClientError("Provided encryption context does not match information retrieved from S3") + raise S3EncryptionClientError( + "Provided encryption context does not match information retrieved from S3" + ) # Update decMaterials with the modified encryption context elif edk.key_provider_info == "kms": if not self.enable_legacy_wrapping_algorithms: - raise S3EncryptionClientError(f"Enable legacy wrapping algorithms to use legacy key wrapping algorithm: {edk.key_provider_info}") + raise S3EncryptionClientError( + f"Enable legacy wrapping algorithms to use legacy key wrapping algorithm: {edk.key_provider_info}" + ) else: - raise S3EncryptionClientError(f"{edk.key_provider_info} is not a valid key wrapping algorithm!") + raise S3EncryptionClientError( + f"{edk.key_provider_info} is not a valid key wrapping algorithm!" + ) response = self.kms_client.decrypt( - KeyId = self.kms_key_id, - CiphertextBlob = edk_bytes, - EncryptionContext = decMaterials.encryption_context_stored + KeyId=self.kms_key_id, + CiphertextBlob=edk_bytes, + EncryptionContext=decMaterials.encryption_context_stored, ) - decMaterials.plaintext_data_key = response['Plaintext'] + decMaterials.plaintext_data_key = response["Plaintext"] return decMaterials except Exception as e: last_exception = e continue - + # If we get here, none of the EDKs could be decrypted if last_exception: raise last_exception diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index f532dfb5..ffab848d 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -1,61 +1,65 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +from typing import Any, Dict, List, Optional + from attrs import define, field -from typing import Optional, Dict, Any, List + from .encrypted_data_key import EncryptedDataKey + @define class EncryptionMaterials: """ Class representing encryption materials for S3 encryption. - + This class provides a structured way to handle encryption materials with fields corresponding to the data needed for encryption operations. - + Attributes: encryption_context (Dict[str, str]): Context information for encryption encrypted_data_key (Optional[EncryptedDataKey]): The encrypted data key plaintext_data_key (Optional[bytes]): The plaintext data key (PDK) """ + encryption_context: Dict[str, str] = field(factory=dict) encrypted_data_key: Optional[EncryptedDataKey] = field(default=None) plaintext_data_key: Optional[bytes] = field(default=None) - + @classmethod - def from_dict(cls, materials_dict: Dict[str, Any]) -> 'EncryptionMaterials': + def from_dict(cls, materials_dict: Dict[str, Any]) -> "EncryptionMaterials": """ Create an EncryptionMaterials instance from a dictionary. - + Args: materials_dict (Dict[str, Any]): Dictionary containing encryption materials - + Returns: EncryptionMaterials: A new instance with fields populated from the dictionary """ return cls( - encryption_context=materials_dict.get('encryption_context', {}), - encrypted_data_key=materials_dict.get('encrypted_data_key'), - plaintext_data_key=materials_dict.get('PDK') + encryption_context=materials_dict.get("encryption_context", {}), + encrypted_data_key=materials_dict.get("encrypted_data_key"), + plaintext_data_key=materials_dict.get("PDK"), ) - + def to_dict(self) -> Dict[str, Any]: """ Convert the EncryptionMaterials instance to a dictionary. - + Returns: Dict[str, Any]: Dictionary containing encryption materials """ result = {} - + if self.encryption_context: - result['encryption_context'] = self.encryption_context - + result["encryption_context"] = self.encryption_context + if self.encrypted_data_key is not None: - result['encrypted_data_key'] = self.encrypted_data_key - + result["encrypted_data_key"] = self.encrypted_data_key + if self.plaintext_data_key is not None: - result['PDK'] = self.plaintext_data_key - + result["PDK"] = self.plaintext_data_key + return result @@ -63,10 +67,10 @@ def to_dict(self) -> Dict[str, Any]: class DecryptionMaterials: """ Class representing decryption materials for S3 encryption. - + This class provides a structured way to handle decryption materials with fields corresponding to the data needed for decryption operations. - + Attributes: iv (Optional[bytes]): The initialization vector used for content encryption encrypted_data_keys (List[EncryptedDataKey]): List of encrypted data keys to try @@ -74,53 +78,56 @@ class DecryptionMaterials: encryption_context_from_request (Dict[str, str]): Encryption context provided in the request plaintext_data_key (Optional[bytes]): The plaintext data key (PDK) """ + iv: Optional[bytes] = field(default=None) encrypted_data_keys: List[EncryptedDataKey] = field(factory=list) encryption_context_stored: Dict[str, str] = field(factory=dict) encryption_context_from_request: Dict[str, str] = field(factory=dict) plaintext_data_key: Optional[bytes] = field(default=None) - + @classmethod - def from_dict(cls, materials_dict: Dict[str, Any]) -> 'DecryptionMaterials': + def from_dict(cls, materials_dict: Dict[str, Any]) -> "DecryptionMaterials": """ Create a DecryptionMaterials instance from a dictionary. - + Args: materials_dict (Dict[str, Any]): Dictionary containing decryption materials - + Returns: DecryptionMaterials: A new instance with fields populated from the dictionary """ return cls( - iv=materials_dict.get('iv'), - encrypted_data_keys=materials_dict.get('encrypted_data_keys', []), - encryption_context_stored=materials_dict.get('encryption_context_stored', {}), - encryption_context_from_request=materials_dict.get('encryption_context_from_request', {}), - plaintext_data_key=materials_dict.get('PDK') + iv=materials_dict.get("iv"), + encrypted_data_keys=materials_dict.get("encrypted_data_keys", []), + encryption_context_stored=materials_dict.get("encryption_context_stored", {}), + encryption_context_from_request=materials_dict.get( + "encryption_context_from_request", {} + ), + plaintext_data_key=materials_dict.get("PDK"), ) - + def to_dict(self) -> Dict[str, Any]: """ Convert the DecryptionMaterials instance to a dictionary. - + Returns: Dict[str, Any]: Dictionary containing decryption materials """ result = {} - + if self.iv is not None: - result['iv'] = self.iv - + result["iv"] = self.iv + if self.encrypted_data_keys: - result['encrypted_data_keys'] = self.encrypted_data_keys - + result["encrypted_data_keys"] = self.encrypted_data_keys + if self.encryption_context_stored: - result['encryption_context_stored'] = self.encryption_context_stored - + result["encryption_context_stored"] = self.encryption_context_stored + if self.encryption_context_from_request: - result['encryption_context_from_request'] = self.encryption_context_from_request - + result["encryption_context_from_request"] = self.encryption_context_from_request + if self.plaintext_data_key is not None: - result['PDK'] = self.plaintext_data_key - + result["PDK"] = self.plaintext_data_key + return result diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py index 4894792a..ce963328 100644 --- a/src/s3_encryption/metadata.py +++ b/src/s3_encryption/metadata.py @@ -1,18 +1,19 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 import json +from typing import Any, Dict, Optional + from attrs import define, field -from typing import Optional, Dict, Any @define class ObjectMetadata: """ Class representing metadata for encrypted S3 objects. - + This class provides a structured way to handle encryption metadata with fields corresponding to standard S3 encryption headers. - + All fields are optional and correspond to the following S3 encryption headers: - encrypted_data_key_v1: The encrypted data key (legacy format) - encrypted_data_key_v2: The encrypted data key (current format) @@ -23,6 +24,7 @@ class ObjectMetadata: - content_cipher_tag_length: The length of the authentication tag - instruction_file: Marker for instruction files """ + # The encrypted data key (legacy format) encrypted_data_key_v1: Optional[str] = field(default=None) # The encrypted data key (current format) @@ -51,13 +53,13 @@ class ObjectMetadata: INSTRUCTION_FILE = "x-amz-crypto-instr-file" @classmethod - def from_dict(cls, metadata_dict: Dict[str, Any]) -> 'ObjectMetadata': + def from_dict(cls, metadata_dict: Dict[str, Any]) -> "ObjectMetadata": """ Create an ObjectMetadata instance from a dictionary. - + Args: metadata_dict (Dict[str, Any]): Dictionary containing metadata keys and values - + Returns: ObjectMetadata: A new instance with fields populated from the dictionary """ @@ -67,7 +69,7 @@ def from_dict(cls, metadata_dict: Dict[str, Any]) -> 'ObjectMetadata': context_str = metadata_dict.get(cls.ENCRYPTED_DATA_KEY_CONTEXT) if context_str is not None: encryption_context = json.loads(context_str) - + return cls( encrypted_data_key_v1=metadata_dict.get(cls.ENCRYPTED_DATA_KEY_V1), encrypted_data_key_v2=metadata_dict.get(cls.ENCRYPTED_DATA_KEY_V2), @@ -76,40 +78,40 @@ def from_dict(cls, metadata_dict: Dict[str, Any]) -> 'ObjectMetadata': content_iv=metadata_dict.get(cls.CONTENT_IV), content_cipher=metadata_dict.get(cls.CONTENT_CIPHER), content_cipher_tag_length=metadata_dict.get(cls.CONTENT_CIPHER_TAG_LENGTH), - instruction_file=metadata_dict.get(cls.INSTRUCTION_FILE) + instruction_file=metadata_dict.get(cls.INSTRUCTION_FILE), ) def to_dict(self) -> Dict[str, str]: """ Convert the ObjectMetadata instance to a dictionary. - + Returns: Dict[str, str]: Dictionary containing non-None metadata values """ result = {} - + if self.encrypted_data_key_v1 is not None: result[self.ENCRYPTED_DATA_KEY_V1] = self.encrypted_data_key_v1 - + if self.encrypted_data_key_v2 is not None: result[self.ENCRYPTED_DATA_KEY_V2] = self.encrypted_data_key_v2 - + if self.encrypted_data_key_algorithm is not None: result[self.ENCRYPTED_DATA_KEY_ALGORITHM] = self.encrypted_data_key_algorithm - + if self.encrypted_data_key_context is not None: result[self.ENCRYPTED_DATA_KEY_CONTEXT] = json.dumps(self.encrypted_data_key_context) - + if self.content_iv is not None: result[self.CONTENT_IV] = self.content_iv - + if self.content_cipher is not None: result[self.CONTENT_CIPHER] = self.content_cipher - + if self.content_cipher_tag_length is not None: result[self.CONTENT_CIPHER_TAG_LENGTH] = self.content_cipher_tag_length - + if self.instruction_file is not None: result[self.INSTRUCTION_FILE] = self.instruction_file - + return result diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 41ec53dc..c207fd45 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -1,33 +1,37 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from attrs import define, field -import os import base64 +import os +from typing import Any, Dict, List, Optional, Union + +from attrs import define, field +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + from .materials.crypto_materials_manager import AbstractCryptoMaterialsManager from .materials.encrypted_data_key import EncryptedDataKey -from .materials.materials import EncryptionMaterials, DecryptionMaterials -from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from .materials.materials import DecryptionMaterials, EncryptionMaterials from .metadata import ObjectMetadata -from typing import Dict, Any, Optional, List, Union + @define class PutEncryptedObjectPipeline: """ Pipeline for encrypting objects before they are put into S3. - + This pipeline handles only the encryption process for S3 objects. The actual S3 API calls are handled by the S3EncryptionClient. """ + cmm: AbstractCryptoMaterialsManager = field() - + def encrypt(self, plaintext, encryption_context=None): """ Encrypt the data before it is stored in S3. - + Args: data (bytes or str): The data to be encrypted encryption_context (dict, optional): Additional context for encryption - + Returns: bytes: The encrypted data dict: Metadata about the encryption to be stored with the object @@ -36,45 +40,41 @@ def encrypt(self, plaintext, encryption_context=None): enc_mats_request = EncryptionMaterials( encryption_context={} if encryption_context is None else encryption_context ) - + # Get encryption materials from the crypto materials manager enc_mats = self.cmm.getEncryptionMaterials(enc_mats_request) - + # Generate initialization vector iv = os.urandom(12) - + # Encrypt the data if enc_mats.plaintext_data_key is None: raise RuntimeError("No plaintext data key found!") - + aesgcm = AESGCM(enc_mats.plaintext_data_key) - ciphertext = aesgcm.encrypt( - nonce=iv, - data=plaintext, - associated_data=None - ) + ciphertext = aesgcm.encrypt(nonce=iv, data=plaintext, associated_data=None) encrypted_data = ciphertext - b64_iv = base64.b64encode(iv).decode('utf-8') - + b64_iv = base64.b64encode(iv).decode("utf-8") + # Get the encrypted data key if enc_mats.encrypted_data_key is None: raise RuntimeError("No encrypted data key found!") - + edk_bytes = enc_mats.encrypted_data_key.encrypted_data_key - b64_edk = base64.b64encode(edk_bytes).decode('utf-8') - + b64_edk = base64.b64encode(edk_bytes).decode("utf-8") + # Create metadata using the ObjectMetadata class metadata = ObjectMetadata( encrypted_data_key_v2=b64_edk, encrypted_data_key_algorithm="kms+context", content_iv=b64_iv, content_cipher="AES/GCM/NoPadding", - encrypted_data_key_context=enc_mats.encryption_context + encrypted_data_key_context=enc_mats.encryption_context, ) - + # Convert to dictionary for storage in S3 metadata encryption_metadata = metadata.to_dict() - + return encrypted_data, encryption_metadata @@ -82,73 +82,70 @@ def encrypt(self, plaintext, encryption_context=None): class GetEncryptedObjectPipeline: """ Pipeline for decrypting objects after they are retrieved from S3. - + This pipeline handles only the decryption process for S3 objects. The actual S3 API calls are handled by the S3EncryptionClient. """ + cmm: AbstractCryptoMaterialsManager = field() - + def decrypt(self, response, encryption_context={}): """ Decrypt the data after it is retrieved from S3. - + Args: response (dict): The response from S3 containing the encrypted data and metadata encryption_context (dict, optional): Additional context for decryption - + Returns: bytes: The decrypted data """ # Convert the metadata dictionary to an ObjectMetadata instance - encrypted_data = response.get('Body').read() - encryption_metadata = response.get('Metadata', {}) + encrypted_data = response.get("Body").read() + encryption_metadata = response.get("Metadata", {}) metadata = ObjectMetadata.from_dict(encryption_metadata) - + iv_b64 = metadata.content_iv edk_b64 = metadata.encrypted_data_key_v2 - + # TODO: probably move this to ObjectMetadata iv_bytes = base64.b64decode(iv_b64) - + # Create a list of encrypted data keys to try encrypted_data_keys = [] # Create an instance of EncryptedDataKey if edk_b64: edk_bytes = base64.b64decode(edk_b64) encrypted_data_key = EncryptedDataKey( - key_provider_id=b'S3Keyring', + key_provider_id=b"S3Keyring", key_provider_info=metadata.encrypted_data_key_algorithm, - encrypted_data_key=edk_bytes + encrypted_data_key=edk_bytes, ) encrypted_data_keys.append(encrypted_data_key) - + # Also check for legacy encrypted data key (v1) if available if metadata.encrypted_data_key_v1: legacy_edk_bytes = base64.b64decode(metadata.encrypted_data_key_v1) legacy_encrypted_data_key = EncryptedDataKey( - key_provider_id=b'S3Keyring', + key_provider_id=b"S3Keyring", key_provider_info=metadata.encrypted_data_key_algorithm, - encrypted_data_key=legacy_edk_bytes + encrypted_data_key=legacy_edk_bytes, ) encrypted_data_keys.append(legacy_encrypted_data_key) - + # Create a DecryptionMaterials instance dec_materials = DecryptionMaterials( iv=iv_bytes, encrypted_data_keys=encrypted_data_keys, encryption_context_stored=metadata.encrypted_data_key_context or {}, - encryption_context_from_request=encryption_context or {} + encryption_context_from_request=encryption_context or {}, ) - + # Get decryption materials from the crypto materials manager dec_materials = self.cmm.decryptMaterials(dec_materials) - + aesgcm = AESGCM(dec_materials.plaintext_data_key) - plaintext = aesgcm.decrypt( - nonce=iv_bytes, - data=encrypted_data, - associated_data=None - ) - + plaintext = aesgcm.decrypt(nonce=iv_bytes, data=encrypted_data, associated_data=None) + return plaintext diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index 55b1d678..d956f321 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -1,14 +1,19 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -import boto3 import os from datetime import datetime + +import boto3 + from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.materials.kms_keyring import KmsKeyring bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") region = os.environ.get("CI_AWS_REGION", "us-west-2") -kms_key_id = os.environ.get("CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key") +kms_key_id = os.environ.get( + "CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" +) + def test_simple_roundtrip(): key = "simple-rt" @@ -24,12 +29,9 @@ def test_simple_roundtrip(): config = S3EncryptionClientConfig(keyring) s3ec = S3EncryptionClient(wrapped_client, config) s3ec.put_object(Bucket=bucket, Key=key, Data=data) - get_req = { - 'Bucket': bucket, - 'Key': key - } + get_req = {"Bucket": bucket, "Key": key} response = s3ec.get_object(**get_req) - output = response['Body'].read().decode('utf-8') + output = response["Body"].read().decode("utf-8") if output != data: print("Uh oh! Input and output don't match!") print("Input:") @@ -37,5 +39,5 @@ def test_simple_roundtrip(): print("Output:") print(output) raise RuntimeError - else: + else: print("Success!") diff --git a/test/test_decryption_materials.py b/test/test_decryption_materials.py index d6c08530..2ada8af8 100644 --- a/test/test_decryption_materials.py +++ b/test/test_decryption_materials.py @@ -2,8 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 import unittest -from src.s3_encryption.materials.materials import DecryptionMaterials + from src.s3_encryption.materials.encrypted_data_key import EncryptedDataKey +from src.s3_encryption.materials.materials import DecryptionMaterials + class TestDecryptionMaterials(unittest.TestCase): def test_create_decryption_materials(self): @@ -14,76 +16,77 @@ def test_create_decryption_materials(self): self.assertEqual(materials.encryption_context_from_request, {}) self.assertIsNone(materials.iv) self.assertIsNone(materials.plaintext_data_key) - + def test_create_with_parameters(self): """Test creating a DecryptionMaterials instance with parameters.""" - iv = b'initialization-vector' + iv = b"initialization-vector" encrypted_data_keys = [ EncryptedDataKey( - key_provider_id=b'S3Keyring', + key_provider_id=b"S3Keyring", key_provider_info="kms+context", - encrypted_data_key=b'encrypted-data-key' + encrypted_data_key=b"encrypted-data-key", ) ] encryption_context_stored = {"key1": "value1"} encryption_context_from_request = {"key2": "value2"} - plaintext_data_key = b'plaintext-data-key' - + plaintext_data_key = b"plaintext-data-key" + materials = DecryptionMaterials( iv=iv, encrypted_data_keys=encrypted_data_keys, encryption_context_stored=encryption_context_stored, encryption_context_from_request=encryption_context_from_request, - plaintext_data_key=plaintext_data_key + plaintext_data_key=plaintext_data_key, ) - + self.assertEqual(materials.iv, iv) self.assertEqual(materials.encrypted_data_keys, encrypted_data_keys) self.assertEqual(materials.encryption_context_stored, encryption_context_stored) self.assertEqual(materials.encryption_context_from_request, encryption_context_from_request) self.assertEqual(materials.plaintext_data_key, plaintext_data_key) - + def test_from_dict(self): """Test creating a DecryptionMaterials instance from a dictionary.""" edk = EncryptedDataKey( - key_provider_id=b'S3Keyring', + key_provider_id=b"S3Keyring", key_provider_info="kms+context", - encrypted_data_key=b'encrypted-data-key' + encrypted_data_key=b"encrypted-data-key", ) materials_dict = { - 'iv': b'initialization-vector', - 'encrypted_data_keys': [edk], - 'encryption_context_stored': {"key1": "value1"}, - 'encryption_context_from_request': {"key2": "value2"}, - 'PDK': b'plaintext-data-key' + "iv": b"initialization-vector", + "encrypted_data_keys": [edk], + "encryption_context_stored": {"key1": "value1"}, + "encryption_context_from_request": {"key2": "value2"}, + "PDK": b"plaintext-data-key", } materials = DecryptionMaterials.from_dict(materials_dict) - self.assertEqual(materials.iv, b'initialization-vector') + self.assertEqual(materials.iv, b"initialization-vector") self.assertEqual(materials.encrypted_data_keys, [edk]) self.assertEqual(materials.encryption_context_stored, {"key1": "value1"}) self.assertEqual(materials.encryption_context_from_request, {"key2": "value2"}) - self.assertEqual(materials.plaintext_data_key, b'plaintext-data-key') - + self.assertEqual(materials.plaintext_data_key, b"plaintext-data-key") + def test_to_dict(self): """Test converting a DecryptionMaterials instance to a dictionary.""" edk = EncryptedDataKey( - key_provider_id=b'S3Keyring', + key_provider_id=b"S3Keyring", key_provider_info="kms+context", - encrypted_data_key=b'encrypted-data-key' + encrypted_data_key=b"encrypted-data-key", ) materials = DecryptionMaterials( - iv=b'initialization-vector', + iv=b"initialization-vector", encrypted_data_keys=[edk], encryption_context_stored={"key1": "value1"}, encryption_context_from_request={"key2": "value2"}, - plaintext_data_key=b'plaintext-data-key' + plaintext_data_key=b"plaintext-data-key", ) materials_dict = materials.to_dict() - self.assertEqual(materials_dict['iv'], b'initialization-vector') - self.assertEqual(materials_dict['encrypted_data_keys'], [edk]) - self.assertEqual(materials_dict['encryption_context_stored'], {"key1": "value1"}) - self.assertEqual(materials_dict['encryption_context_from_request'], {"key2": "value2"}) - self.assertEqual(materials_dict['PDK'], b'plaintext-data-key') + self.assertEqual(materials_dict["iv"], b"initialization-vector") + self.assertEqual(materials_dict["encrypted_data_keys"], [edk]) + self.assertEqual(materials_dict["encryption_context_stored"], {"key1": "value1"}) + self.assertEqual(materials_dict["encryption_context_from_request"], {"key2": "value2"}) + self.assertEqual(materials_dict["PDK"], b"plaintext-data-key") + -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/test/test_decryption_materials_integration.py b/test/test_decryption_materials_integration.py index dbbaf431..7e697e92 100644 --- a/test/test_decryption_materials_integration.py +++ b/test/test_decryption_materials_integration.py @@ -3,148 +3,153 @@ import unittest from unittest.mock import MagicMock, patch -from src.s3_encryption.materials.materials import DecryptionMaterials + +from src.s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager from src.s3_encryption.materials.encrypted_data_key import EncryptedDataKey from src.s3_encryption.materials.keyring import S3Keyring -from src.s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager +from src.s3_encryption.materials.materials import DecryptionMaterials + class TestDecryptionMaterialsIntegration(unittest.TestCase): def test_keyring_onDecrypt(self): """Test that S3Keyring.onDecrypt properly handles DecryptionMaterials.""" # Create a keyring keyring = S3Keyring() - + # Create an encrypted data key edk = EncryptedDataKey( - key_provider_id=b'S3Keyring', + key_provider_id=b"S3Keyring", key_provider_info="kms+context", - encrypted_data_key=b'encrypted-data-key' + encrypted_data_key=b"encrypted-data-key", ) - + # Create decryption materials materials = DecryptionMaterials( - iv=b'initialization-vector', + iv=b"initialization-vector", encrypted_data_keys=[edk], encryption_context_stored={"key1": "value1"}, - encryption_context_from_request={"key2": "value2"} + encryption_context_from_request={"key2": "value2"}, ) - + # Mock the validation method to return the materials - with patch.object(S3Keyring, 'onDecrypt', return_value=materials) as mock_onDecrypt: + with patch.object(S3Keyring, "onDecrypt", return_value=materials) as mock_onDecrypt: # Call onDecrypt result = keyring.onDecrypt(materials, [edk]) - + # Verify the result is a DecryptionMaterials instance self.assertIsInstance(result, DecryptionMaterials) - self.assertEqual(result.iv, b'initialization-vector') + self.assertEqual(result.iv, b"initialization-vector") self.assertEqual(result.encrypted_data_keys, [edk]) self.assertEqual(result.encryption_context_stored, {"key1": "value1"}) self.assertEqual(result.encryption_context_from_request, {"key2": "value2"}) - + def test_keyring_onDecrypt_default_EC(self): """Test that S3Keyring.onDecrypt properly handles DecryptionMaterials.""" # Create a keyring keyring = S3Keyring() - + # Create an encrypted data key edk = EncryptedDataKey( - key_provider_id=b'S3Keyring', + key_provider_id=b"S3Keyring", key_provider_info="kms+context", - encrypted_data_key=b'encrypted-data-key' + encrypted_data_key=b"encrypted-data-key", ) - + # Create decryption materials materials = DecryptionMaterials( - iv=b'initialization-vector', + iv=b"initialization-vector", encrypted_data_keys=[edk], encryption_context_stored={}, - encryption_context_from_request={} + encryption_context_from_request={}, ) - + # Mock the validation method to return the materials - with patch.object(S3Keyring, 'onDecrypt', return_value=materials) as mock_onDecrypt: + with patch.object(S3Keyring, "onDecrypt", return_value=materials) as mock_onDecrypt: # Call onDecrypt result = keyring.onDecrypt(materials, [edk]) - + # Verify the result is a DecryptionMaterials instance self.assertIsInstance(result, DecryptionMaterials) - self.assertEqual(result.iv, b'initialization-vector') + self.assertEqual(result.iv, b"initialization-vector") self.assertEqual(result.encrypted_data_keys, [edk]) self.assertEqual(result.encryption_context_stored, {}) self.assertEqual(result.encryption_context_from_request, {}) - + def test_cmm_decryptMaterials_with_dict(self): """Test that DefaultCryptoMaterialsManager.decryptMaterials properly handles dictionary input.""" # Create a mock keyring keyring = MagicMock() edk = EncryptedDataKey( - key_provider_id=b'S3Keyring', + key_provider_id=b"S3Keyring", key_provider_info="kms+context", - encrypted_data_key=b'encrypted-data-key' + encrypted_data_key=b"encrypted-data-key", ) keyring.onDecrypt.return_value = DecryptionMaterials( - iv=b'initialization-vector', + iv=b"initialization-vector", encrypted_data_keys=[edk], encryption_context_stored={"key1": "value1"}, encryption_context_from_request={"key2": "value2"}, - plaintext_data_key=b'plaintext-data-key' + plaintext_data_key=b"plaintext-data-key", ) - + # Create a CMM cmm = DefaultCryptoMaterialsManager(keyring=keyring) - + # Call decryptMaterials with a dictionary - result = cmm.decryptMaterials({ - 'iv': b'initialization-vector', - 'encrypted_data_keys': [edk], - 'encryption_context_stored': {"key1": "value1"}, - 'encryption_context_from_request': {"key2": "value2"} - }) - + result = cmm.decryptMaterials( + { + "iv": b"initialization-vector", + "encrypted_data_keys": [edk], + "encryption_context_stored": {"key1": "value1"}, + "encryption_context_from_request": {"key2": "value2"}, + } + ) + # Verify the result is a DecryptionMaterials instance self.assertIsInstance(result, DecryptionMaterials) - self.assertEqual(result.iv, b'initialization-vector') + self.assertEqual(result.iv, b"initialization-vector") self.assertEqual(result.encrypted_data_keys, [edk]) self.assertEqual(result.encryption_context_stored, {"key1": "value1"}) self.assertEqual(result.encryption_context_from_request, {"key2": "value2"}) - self.assertEqual(result.plaintext_data_key, b'plaintext-data-key') - + self.assertEqual(result.plaintext_data_key, b"plaintext-data-key") + def test_cmm_decryptMaterials_with_materials(self): """Test that DefaultCryptoMaterialsManager.decryptMaterials properly handles DecryptionMaterials input.""" # Create a mock keyring keyring = MagicMock() edk = EncryptedDataKey( - key_provider_id=b'S3Keyring', + key_provider_id=b"S3Keyring", key_provider_info="kms+context", - encrypted_data_key=b'encrypted-data-key' + encrypted_data_key=b"encrypted-data-key", ) keyring.onDecrypt.return_value = DecryptionMaterials( - iv=b'initialization-vector', + iv=b"initialization-vector", encrypted_data_keys=[edk], encryption_context_stored={"key1": "value1"}, encryption_context_from_request={"key2": "value2"}, - plaintext_data_key=b'plaintext-data-key' + plaintext_data_key=b"plaintext-data-key", ) - + # Create a CMM cmm = DefaultCryptoMaterialsManager(keyring=keyring) - + # Call decryptMaterials with a DecryptionMaterials instance materials = DecryptionMaterials( - iv=b'initialization-vector', + iv=b"initialization-vector", encrypted_data_keys=[edk], encryption_context_stored={"key1": "value1"}, - encryption_context_from_request={"key2": "value2"} + encryption_context_from_request={"key2": "value2"}, ) result = cmm.decryptMaterials(materials) - + # Verify the result is a DecryptionMaterials instance self.assertIsInstance(result, DecryptionMaterials) - self.assertEqual(result.iv, b'initialization-vector') + self.assertEqual(result.iv, b"initialization-vector") self.assertEqual(result.encrypted_data_keys, [edk]) self.assertEqual(result.encryption_context_stored, {"key1": "value1"}) self.assertEqual(result.encryption_context_from_request, {"key2": "value2"}) - self.assertEqual(result.plaintext_data_key, b'plaintext-data-key') + self.assertEqual(result.plaintext_data_key, b"plaintext-data-key") + -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/test/test_encryption_materials.py b/test/test_encryption_materials.py index b1e6b653..ffa5a449 100644 --- a/test/test_encryption_materials.py +++ b/test/test_encryption_materials.py @@ -2,8 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 import unittest -from src.s3_encryption.materials.materials import EncryptionMaterials + from src.s3_encryption.materials.encrypted_data_key import EncryptedDataKey +from src.s3_encryption.materials.materials import EncryptionMaterials + class TestEncryptionMaterials(unittest.TestCase): def test_create_encryption_materials(self): @@ -12,46 +14,47 @@ def test_create_encryption_materials(self): self.assertEqual(materials.encryption_context, {}) self.assertIsNone(materials.encrypted_data_key) self.assertIsNone(materials.plaintext_data_key) - + def test_create_with_encryption_context(self): """Test creating an EncryptionMaterials instance with an encryption context.""" encryption_context = {"key1": "value1", "key2": "value2"} materials = EncryptionMaterials(encryption_context=encryption_context) self.assertEqual(materials.encryption_context, encryption_context) - + def test_from_dict(self): """Test creating an EncryptionMaterials instance from a dictionary.""" edk = EncryptedDataKey( - key_provider_id=b'S3Keyring', + key_provider_id=b"S3Keyring", key_provider_info="kms+context", - encrypted_data_key=b'encrypted-data-key' + encrypted_data_key=b"encrypted-data-key", ) materials_dict = { - 'encryption_context': {"key1": "value1"}, - 'encrypted_data_key': edk, - 'PDK': b'plaintext-data-key' + "encryption_context": {"key1": "value1"}, + "encrypted_data_key": edk, + "PDK": b"plaintext-data-key", } materials = EncryptionMaterials.from_dict(materials_dict) self.assertEqual(materials.encryption_context, {"key1": "value1"}) self.assertEqual(materials.encrypted_data_key, edk) - self.assertEqual(materials.plaintext_data_key, b'plaintext-data-key') - + self.assertEqual(materials.plaintext_data_key, b"plaintext-data-key") + def test_to_dict(self): """Test converting an EncryptionMaterials instance to a dictionary.""" edk = EncryptedDataKey( - key_provider_id=b'S3Keyring', + key_provider_id=b"S3Keyring", key_provider_info="kms+context", - encrypted_data_key=b'encrypted-data-key' + encrypted_data_key=b"encrypted-data-key", ) materials = EncryptionMaterials( encryption_context={"key1": "value1"}, encrypted_data_key=edk, - plaintext_data_key=b'plaintext-data-key' + plaintext_data_key=b"plaintext-data-key", ) materials_dict = materials.to_dict() - self.assertEqual(materials_dict['encryption_context'], {"key1": "value1"}) - self.assertEqual(materials_dict['encrypted_data_key'], edk) - self.assertEqual(materials_dict['PDK'], b'plaintext-data-key') + self.assertEqual(materials_dict["encryption_context"], {"key1": "value1"}) + self.assertEqual(materials_dict["encrypted_data_key"], edk) + self.assertEqual(materials_dict["PDK"], b"plaintext-data-key") + -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/test/test_encryption_materials_integration.py b/test/test_encryption_materials_integration.py index 7082bb8f..da8a17b2 100644 --- a/test/test_encryption_materials_integration.py +++ b/test/test_encryption_materials_integration.py @@ -3,29 +3,29 @@ import unittest from unittest.mock import MagicMock, patch -from src.s3_encryption.materials.materials import EncryptionMaterials + +from src.s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager from src.s3_encryption.materials.encrypted_data_key import EncryptedDataKey from src.s3_encryption.materials.keyring import S3Keyring -from src.s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager +from src.s3_encryption.materials.materials import EncryptionMaterials + class TestEncryptionMaterialsIntegration(unittest.TestCase): def test_keyring_onEncrypt(self): """Test that S3Keyring.onEncrypt properly handles EncryptionMaterials.""" # Create a keyring keyring = S3Keyring() - + # Create encryption materials - materials = EncryptionMaterials( - encryption_context={"key1": "value1"} - ) - + materials = EncryptionMaterials(encryption_context={"key1": "value1"}) + # Call onEncrypt result = keyring.onEncrypt(materials) - + # Verify the result is an EncryptionMaterials instance self.assertIsInstance(result, EncryptionMaterials) self.assertEqual(result.encryption_context, {"key1": "value1"}) - + def test_cmm_getEncryptionMaterials_with_dict(self): """Test that DefaultCryptoMaterialsManager.getEncryptionMaterials properly handles dictionary input.""" # Create a mock keyring @@ -33,25 +33,25 @@ def test_cmm_getEncryptionMaterials_with_dict(self): keyring.onEncrypt.return_value = EncryptionMaterials( encryption_context={"key1": "value1"}, encrypted_data_key=EncryptedDataKey( - key_provider_id=b'S3Keyring', + key_provider_id=b"S3Keyring", key_provider_info="kms+context", - encrypted_data_key=b'encrypted-data-key' + encrypted_data_key=b"encrypted-data-key", ), - plaintext_data_key=b'plaintext-data-key' + plaintext_data_key=b"plaintext-data-key", ) - + # Create a CMM cmm = DefaultCryptoMaterialsManager(keyring=keyring) - + # Call getEncryptionMaterials with a dictionary result = cmm.getEncryptionMaterials({"encryption_context": {"key1": "value1"}}) - + # Verify the result is an EncryptionMaterials instance self.assertIsInstance(result, EncryptionMaterials) self.assertEqual(result.encryption_context, {"key1": "value1"}) self.assertIsNotNone(result.encrypted_data_key) self.assertIsNotNone(result.plaintext_data_key) - + def test_cmm_getEncryptionMaterials_with_materials(self): """Test that DefaultCryptoMaterialsManager.getEncryptionMaterials properly handles EncryptionMaterials input.""" # Create a mock keyring @@ -59,27 +59,26 @@ def test_cmm_getEncryptionMaterials_with_materials(self): keyring.onEncrypt.return_value = EncryptionMaterials( encryption_context={"key1": "value1"}, encrypted_data_key=EncryptedDataKey( - key_provider_id=b'S3Keyring', + key_provider_id=b"S3Keyring", key_provider_info="kms+context", - encrypted_data_key=b'encrypted-data-key' + encrypted_data_key=b"encrypted-data-key", ), - plaintext_data_key=b'plaintext-data-key' + plaintext_data_key=b"plaintext-data-key", ) - + # Create a CMM cmm = DefaultCryptoMaterialsManager(keyring=keyring) - + # Call getEncryptionMaterials with an EncryptionMaterials instance - materials = EncryptionMaterials( - encryption_context={"key1": "value1"} - ) + materials = EncryptionMaterials(encryption_context={"key1": "value1"}) result = cmm.getEncryptionMaterials(materials) - + # Verify the result is an EncryptionMaterials instance self.assertIsInstance(result, EncryptionMaterials) self.assertEqual(result.encryption_context, {"key1": "value1"}) self.assertIsNotNone(result.encrypted_data_key) self.assertIsNotNone(result.plaintext_data_key) -if __name__ == '__main__': + +if __name__ == "__main__": unittest.main() diff --git a/test/test_metadata.py b/test/test_metadata.py index 40a813aa..2c26c336 100644 --- a/test/test_metadata.py +++ b/test/test_metadata.py @@ -1,11 +1,11 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -import unittest -import sys import os +import sys +import unittest # Add the src directory to the Python path -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../src'))) +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../src"))) from s3_encryption.metadata import ObjectMetadata @@ -17,67 +17,67 @@ def test_from_dict(self): "x-amz-key-v2": "encrypted-key-data", "x-amz-wrap-alg": "kms+context", "x-amz-iv": "base64-encoded-iv", - "x-amz-cek-alg": "AES/GCM/NoPadding" + "x-amz-cek-alg": "AES/GCM/NoPadding", } - + # Create an ObjectMetadata instance from the dictionary metadata = ObjectMetadata.from_dict(metadata_dict) - + # Verify that the fields were populated correctly self.assertEqual(metadata.encrypted_data_key_v2, "encrypted-key-data") self.assertEqual(metadata.encrypted_data_key_algorithm, "kms+context") self.assertEqual(metadata.content_iv, "base64-encoded-iv") self.assertEqual(metadata.content_cipher, "AES/GCM/NoPadding") - + # Verify that fields not in the dictionary are None self.assertIsNone(metadata.encrypted_data_key_v1) self.assertIsNone(metadata.encrypted_data_key_context) # Note: content_cipher_tag_length is None because it's not in the input dictionary self.assertIsNone(metadata.content_cipher_tag_length) self.assertIsNone(metadata.instruction_file) - + def test_to_dict(self): # Create an ObjectMetadata instance with some fields set metadata = ObjectMetadata( encrypted_data_key_v2="encrypted-key-data", encrypted_data_key_algorithm="kms+context", content_iv="base64-encoded-iv", - content_cipher="AES/GCM/NoPadding" + content_cipher="AES/GCM/NoPadding", ) - + # Convert to dictionary metadata_dict = metadata.to_dict() - + # Verify that the dictionary contains the expected keys and values self.assertEqual(metadata_dict["x-amz-key-v2"], "encrypted-key-data") self.assertEqual(metadata_dict["x-amz-wrap-alg"], "kms+context") self.assertEqual(metadata_dict["x-amz-iv"], "base64-encoded-iv") self.assertEqual(metadata_dict["x-amz-cek-alg"], "AES/GCM/NoPadding") - + # Verify that fields that are None are not included in the dictionary self.assertNotIn("x-amz-key", metadata_dict) self.assertNotIn("x-amz-matdesc", metadata_dict) # Note: content_cipher_tag_length has a default value of "128" self.assertEqual(metadata_dict.get("x-amz-tag-len"), "128") self.assertNotIn("x-amz-crypto-instr-file", metadata_dict) - + def test_roundtrip(self): # Create a metadata dictionary original_dict = { "x-amz-key-v2": "encrypted-key-data", "x-amz-wrap-alg": "kms+context", "x-amz-iv": "base64-encoded-iv", - "x-amz-cek-alg": "AES/GCM/NoPadding" + "x-amz-cek-alg": "AES/GCM/NoPadding", } - + # Convert to ObjectMetadata and back to dictionary metadata = ObjectMetadata.from_dict(original_dict) result_dict = metadata.to_dict() - + # Remove the tag length field which has a default value if "x-amz-tag-len" in result_dict: result_dict.pop("x-amz-tag-len") - + # Verify that the result matches the original self.assertEqual(result_dict, original_dict) From 454c0ebf6d64afff30cd5fccd9d0aed61aed5571 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 11 Aug 2025 13:15:50 -0700 Subject: [PATCH 15/81] address feedback (ruff, uv, etc) --- Makefile | 21 +- pyproject.toml | 62 +-- requirements.txt | 12 + src/s3_encryption/__init__.py | 3 +- .../materials/crypto_materials_manager.py | 13 +- .../materials/encrypted_data_key.py | 3 +- src/s3_encryption/materials/keyring.py | 18 +- src/s3_encryption/materials/kms_keyring.py | 12 +- src/s3_encryption/materials/materials.py | 44 +- src/s3_encryption/metadata.py | 31 +- src/s3_encryption/pipelines.py | 13 +- test/integration/test_i_s3_encryption.py | 1 - test/test_encryption_materials_integration.py | 2 +- uv.lock | 380 ++++++++++++++++++ 14 files changed, 494 insertions(+), 121 deletions(-) create mode 100644 requirements.txt create mode 100644 uv.lock diff --git a/Makefile b/Makefile index 15dd281a..65578384 100644 --- a/Makefile +++ b/Makefile @@ -5,30 +5,31 @@ all: lint test # Install dependencies install: - poetry install + uv pip install -e ".[dev,test]" # Run linting checks lint: - poetry run black --check . - poetry run isort --check . - # Allow flake8 to fail for now as we're gradually adopting linting standards - poetry run flake8 src/ test/ || true + uv run black --check . + uv run isort --check . + # Allow ruff to fail for now as we're gradually adopting linting standards + uv run ruff check src/ test/ || true -# Format code with Black and isort +# Format code with Black, isort, and Ruff format: - poetry run black . - poetry run isort . + uv run black . + uv run isort . + uv run ruff check --fix src/ test/ # Run all tests test: test-unit test-integration # Run unit tests test-unit: - poetry run pytest test/ --verbose + uv run pytest test/ --verbose # Run integration tests test-integration: - poetry run pytest test/integration/ --verbose + uv run pytest test/integration/ --verbose # Clean up cache files clean: diff --git a/pyproject.toml b/pyproject.toml index 89a86e17..a613bc12 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,43 +1,55 @@ -[tool.poetry] +[project] name = "amazon-s3-encryption-client-python" version = "0.1.0" description = "This library provides an S3 client that supports client-side encryption." -authors = ["AWS Crypto Tools "] -license = "Apache-2.0" +authors = [ + {name = "AWS Crypto Tools", email = "aws-crypto-tools@amazon.com"} +] +license = {text = "Apache-2.0"} readme = "README.md" -packages = [{include = "s3_encryption", from = "src"}] - -[tool.poetry.dependencies] -python = "^3.11" -boto3 = "^1.37.2" -# There is a newer version, but MPL wants this one. -cryptography = "^43.0.1" -aws-cryptographic-material-providers = "^1.7.4" -attrs = "^25.1.0" - -[tool.poetry.group.dev.dependencies] -pytest = "^8.4.1" -black = "^24.3.0" -flake8 = "^7.0.0" -flake8-docstrings = "^1.7.0" -isort = "^5.13.2" +requires-python = ">=3.11" +dependencies = [ + "boto3>=1.37.2", + "cryptography>=45.0.6", + "attrs>=25.1.0", +] +[project.optional-dependencies] +test = [ + "pytest>=8.4.1", +] +dev = [ + "black>=24.3.0", + "ruff>=0.3.0", + "isort>=5.13.2", +] [build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/s3_encryption"] [tool.black] line-length = 100 target-version = ["py311"] include = '\.pyi?$' -[tool.flake8] -max-line-length = 100 +[tool.ruff] +line-length = 100 +target-version = "py311" exclude = [".git", "__pycache__", "build", "dist"] + +[tool.ruff.lint] +# Enable all rules by default, then configure specific rule settings below +select = ["E", "F", "W", "I", "N", "D", "UP", "B", "A", "C4", "PT", "RET", "SIM", "ARG", "ERA"] + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.lint.mccabe] max-complexity = 10 -ignore = ["E203", "W503"] # E203 and W503 conflict with Black -docstring-convention = "google" [tool.isort] profile = "black" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..423d8c8d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +# Main dependencies +boto3>=1.37.2 +cryptography>=45.0.6 +attrs>=25.1.0 + +# Test dependencies +pytest>=8.4.1 + +# Development dependencies +black>=24.3.0 +ruff>=0.3.0 +isort>=5.13.2 diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index ab0a77fe..fb633945 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -16,8 +16,7 @@ @define class S3EncryptionClientConfig: - """ - Configuration object for the S3 Encryption Client + """Configuration object for the S3 Encryption Client """ keyring: AbstractKeyring diff --git a/src/s3_encryption/materials/crypto_materials_manager.py b/src/s3_encryption/materials/crypto_materials_manager.py index c2d67edf..c668a53e 100644 --- a/src/s3_encryption/materials/crypto_materials_manager.py +++ b/src/s3_encryption/materials/crypto_materials_manager.py @@ -1,6 +1,5 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from typing import Any, Dict, List, Union from attrs import define @@ -11,8 +10,7 @@ # API Stub for CMM class AbstractCryptoMaterialsManager: def getEncryptionMaterials(self, encMatsRequest): - """ - Get encryption materials from the keyring. + """Get encryption materials from the keyring. Args: encMatsRequest (Dict[str, Any] or EncryptionMaterials): Request containing encryption parameters @@ -23,8 +21,7 @@ def getEncryptionMaterials(self, encMatsRequest): raise NotImplementedError def decryptMaterials(self, decMatsRequest): - """ - Decrypt materials using the keyring. + """Decrypt materials using the keyring. Args: decMatsRequest (Dict[str, Any] or DecryptionMaterials): Request containing decryption parameters @@ -40,8 +37,7 @@ class DefaultCryptoMaterialsManager(AbstractCryptoMaterialsManager): keyring: AbstractKeyring def getEncryptionMaterials(self, encMatsRequest): - """ - Get encryption materials from the keyring. + """Get encryption materials from the keyring. Args: encMatsRequest (Dict[str, Any]): Request containing encryption parameters @@ -60,8 +56,7 @@ def getEncryptionMaterials(self, encMatsRequest): return self.keyring.onEncrypt(materials) def decryptMaterials(self, decMatsRequest): - """ - Decrypt materials using the keyring. + """Decrypt materials using the keyring. Args: decMatsRequest (Dict[str, Any] or DecryptionMaterials): Request containing decryption parameters diff --git a/src/s3_encryption/materials/encrypted_data_key.py b/src/s3_encryption/materials/encrypted_data_key.py index 0dbfa08f..51d47c3b 100644 --- a/src/s3_encryption/materials/encrypted_data_key.py +++ b/src/s3_encryption/materials/encrypted_data_key.py @@ -5,8 +5,7 @@ @define class EncryptedDataKey: - """ - Class representing an encrypted data key. + """Class representing an encrypted data key. An encrypted data key contains information about the key provider and the encrypted data key itself. diff --git a/src/s3_encryption/materials/keyring.py b/src/s3_encryption/materials/keyring.py index 98e4f6ab..f840bce4 100644 --- a/src/s3_encryption/materials/keyring.py +++ b/src/s3_encryption/materials/keyring.py @@ -1,9 +1,8 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from typing import List, Optional -from attrs import define, field +from attrs import define from ..exceptions import S3EncryptionClientError from .materials import DecryptionMaterials, EncryptionMaterials @@ -18,8 +17,7 @@ class AbstractKeyring: # enableLegacyWrappingAlgorithms: bool = field(default=False) def onEncrypt(self, encMaterials): - """ - Process encryption materials. + """Process encryption materials. Args: encMaterials (EncryptionMaterials): Encryption materials to process @@ -30,8 +28,7 @@ def onEncrypt(self, encMaterials): raise NotImplementedError def onDecrypt(self, decMaterials, encrypted_data_keys=None): - """ - Decrypt one of the encrypted data keys and update decMaterials. + """Decrypt one of the encrypted data keys and update decMaterials. Args: decMaterials (DecryptionMaterials): A DecryptionMaterials instance containing decryption materials @@ -45,16 +42,14 @@ def onDecrypt(self, decMaterials, encrypted_data_keys=None): @define class S3Keyring(AbstractKeyring): - """ - Base class for S3 encryption keyrings that provides common validation logic. + """Base class for S3 encryption keyrings that provides common validation logic. """ # Ideally this would be set, but attrs doesn't play nice # enable_legacy_wrapping_algorithms: bool = field(default=False) def onEncrypt(self, encMaterials): - """ - Validate encryption materials before encryption. + """Validate encryption materials before encryption. Args: encMaterials (EncryptionMaterials or dict): Encryption materials @@ -79,8 +74,7 @@ def onEncrypt(self, encMaterials): return encMaterials def onDecrypt(self, decMaterials, encrypted_data_keys=None): - """ - Validate decryption materials before decryption. + """Validate decryption materials before decryption. Args: decMaterials (DecryptionMaterials): A DecryptionMaterials instance containing decryption materials diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index 4ba5e31b..9559ff29 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -1,13 +1,11 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from typing import List, Optional from attrs import define, field from ..exceptions import S3EncryptionClientError from .encrypted_data_key import EncryptedDataKey from .keyring import S3Keyring -from .materials import DecryptionMaterials, EncryptionMaterials KMS_CONTEXT_DEFAULT_KEY = "aws:x-amz-cek-alg" KMS_V1_DEFAULT_KEY = "kms_cmk_id" @@ -20,8 +18,7 @@ class KmsKeyring(S3Keyring): enable_legacy_wrapping_algorithms: bool = field(default=False) def onEncrypt(self, encMaterials): - """ - Process encryption materials using KMS. + """Process encryption materials using KMS. Args: encMaterials (EncryptionMaterials): Encryption materials to process @@ -49,12 +46,11 @@ def onEncrypt(self, encMaterials): encMaterials.encrypted_data_key = encrypted_data_key encMaterials.plaintext_data_key = response["Plaintext"] return encMaterials - except Exception as e: + except Exception: raise def onDecrypt(self, decMaterials, encrypted_data_keys=None): - """ - Decrypt one of the encrypted data keys and update decMaterials. + """Decrypt one of the encrypted data keys and update decMaterials. Args: decMaterials (DecryptionMaterials): A DecryptionMaterials instance containing decryption materials @@ -129,5 +125,5 @@ def onDecrypt(self, decMaterials, encrypted_data_keys=None): raise last_exception else: raise S3EncryptionClientError("Failed to decrypt any of the encrypted data keys") - except Exception as e: + except Exception: raise diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index ffab848d..1ab235da 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -1,6 +1,6 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from typing import Any, Dict, List, Optional +from typing import Any from attrs import define, field @@ -9,8 +9,7 @@ @define class EncryptionMaterials: - """ - Class representing encryption materials for S3 encryption. + """Class representing encryption materials for S3 encryption. This class provides a structured way to handle encryption materials with fields corresponding to the data needed for encryption operations. @@ -21,14 +20,13 @@ class EncryptionMaterials: plaintext_data_key (Optional[bytes]): The plaintext data key (PDK) """ - encryption_context: Dict[str, str] = field(factory=dict) - encrypted_data_key: Optional[EncryptedDataKey] = field(default=None) - plaintext_data_key: Optional[bytes] = field(default=None) + encryption_context: dict[str, str] = field(factory=dict) + encrypted_data_key: EncryptedDataKey | None = field(default=None) + plaintext_data_key: bytes | None = field(default=None) @classmethod - def from_dict(cls, materials_dict: Dict[str, Any]) -> "EncryptionMaterials": - """ - Create an EncryptionMaterials instance from a dictionary. + def from_dict(cls, materials_dict: dict[str, Any]) -> "EncryptionMaterials": + """Create an EncryptionMaterials instance from a dictionary. Args: materials_dict (Dict[str, Any]): Dictionary containing encryption materials @@ -42,9 +40,8 @@ def from_dict(cls, materials_dict: Dict[str, Any]) -> "EncryptionMaterials": plaintext_data_key=materials_dict.get("PDK"), ) - def to_dict(self) -> Dict[str, Any]: - """ - Convert the EncryptionMaterials instance to a dictionary. + def to_dict(self) -> dict[str, Any]: + """Convert the EncryptionMaterials instance to a dictionary. Returns: Dict[str, Any]: Dictionary containing encryption materials @@ -65,8 +62,7 @@ def to_dict(self) -> Dict[str, Any]: @define class DecryptionMaterials: - """ - Class representing decryption materials for S3 encryption. + """Class representing decryption materials for S3 encryption. This class provides a structured way to handle decryption materials with fields corresponding to the data needed for decryption operations. @@ -79,16 +75,15 @@ class DecryptionMaterials: plaintext_data_key (Optional[bytes]): The plaintext data key (PDK) """ - iv: Optional[bytes] = field(default=None) - encrypted_data_keys: List[EncryptedDataKey] = field(factory=list) - encryption_context_stored: Dict[str, str] = field(factory=dict) - encryption_context_from_request: Dict[str, str] = field(factory=dict) - plaintext_data_key: Optional[bytes] = field(default=None) + iv: bytes | None = field(default=None) + encrypted_data_keys: list[EncryptedDataKey] = field(factory=list) + encryption_context_stored: dict[str, str] = field(factory=dict) + encryption_context_from_request: dict[str, str] = field(factory=dict) + plaintext_data_key: bytes | None = field(default=None) @classmethod - def from_dict(cls, materials_dict: Dict[str, Any]) -> "DecryptionMaterials": - """ - Create a DecryptionMaterials instance from a dictionary. + def from_dict(cls, materials_dict: dict[str, Any]) -> "DecryptionMaterials": + """Create a DecryptionMaterials instance from a dictionary. Args: materials_dict (Dict[str, Any]): Dictionary containing decryption materials @@ -106,9 +101,8 @@ def from_dict(cls, materials_dict: Dict[str, Any]) -> "DecryptionMaterials": plaintext_data_key=materials_dict.get("PDK"), ) - def to_dict(self) -> Dict[str, Any]: - """ - Convert the DecryptionMaterials instance to a dictionary. + def to_dict(self) -> dict[str, Any]: + """Convert the DecryptionMaterials instance to a dictionary. Returns: Dict[str, Any]: Dictionary containing decryption materials diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py index ce963328..c59370bb 100644 --- a/src/s3_encryption/metadata.py +++ b/src/s3_encryption/metadata.py @@ -1,15 +1,14 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 import json -from typing import Any, Dict, Optional +from typing import Any from attrs import define, field @define class ObjectMetadata: - """ - Class representing metadata for encrypted S3 objects. + """Class representing metadata for encrypted S3 objects. This class provides a structured way to handle encryption metadata with fields corresponding to standard S3 encryption headers. @@ -26,21 +25,21 @@ class ObjectMetadata: """ # The encrypted data key (legacy format) - encrypted_data_key_v1: Optional[str] = field(default=None) + encrypted_data_key_v1: str | None = field(default=None) # The encrypted data key (current format) - encrypted_data_key_v2: Optional[str] = field(default=None) + encrypted_data_key_v2: str | None = field(default=None) # The algorithm used to encrypt the data key (e.g. AES/GCM or kms+context) - encrypted_data_key_algorithm: Optional[str] = field(default=None) + encrypted_data_key_algorithm: str | None = field(default=None) # The encryption context used for the data key - encrypted_data_key_context: Optional[dict] = field(default=None) + encrypted_data_key_context: dict | None = field(default=None) # The initialization vector used for content encryption - content_iv: Optional[str] = field(default=None) + content_iv: str | None = field(default=None) # The cipher algorithm used for content encryption (e.g. AES/GCM/NoPadding) - content_cipher: Optional[str] = field(default=None) + content_cipher: str | None = field(default=None) # The length of the authentication tag - content_cipher_tag_length: Optional[str] = field(default="128") + content_cipher_tag_length: str | None = field(default="128") # Marker for instruction files - instruction_file: Optional[str] = field(default=None) + instruction_file: str | None = field(default=None) # Constants for metadata keys ENCRYPTED_DATA_KEY_V1 = "x-amz-key" @@ -53,9 +52,8 @@ class ObjectMetadata: INSTRUCTION_FILE = "x-amz-crypto-instr-file" @classmethod - def from_dict(cls, metadata_dict: Dict[str, Any]) -> "ObjectMetadata": - """ - Create an ObjectMetadata instance from a dictionary. + def from_dict(cls, metadata_dict: dict[str, Any]) -> "ObjectMetadata": + """Create an ObjectMetadata instance from a dictionary. Args: metadata_dict (Dict[str, Any]): Dictionary containing metadata keys and values @@ -81,9 +79,8 @@ def from_dict(cls, metadata_dict: Dict[str, Any]) -> "ObjectMetadata": instruction_file=metadata_dict.get(cls.INSTRUCTION_FILE), ) - def to_dict(self) -> Dict[str, str]: - """ - Convert the ObjectMetadata instance to a dictionary. + def to_dict(self) -> dict[str, str]: + """Convert the ObjectMetadata instance to a dictionary. Returns: Dict[str, str]: Dictionary containing non-None metadata values diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index c207fd45..44aa4dfa 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 import base64 import os -from typing import Any, Dict, List, Optional, Union from attrs import define, field from cryptography.hazmat.primitives.ciphers.aead import AESGCM @@ -15,8 +14,7 @@ @define class PutEncryptedObjectPipeline: - """ - Pipeline for encrypting objects before they are put into S3. + """Pipeline for encrypting objects before they are put into S3. This pipeline handles only the encryption process for S3 objects. The actual S3 API calls are handled by the S3EncryptionClient. @@ -25,8 +23,7 @@ class PutEncryptedObjectPipeline: cmm: AbstractCryptoMaterialsManager = field() def encrypt(self, plaintext, encryption_context=None): - """ - Encrypt the data before it is stored in S3. + """Encrypt the data before it is stored in S3. Args: data (bytes or str): The data to be encrypted @@ -80,8 +77,7 @@ def encrypt(self, plaintext, encryption_context=None): @define class GetEncryptedObjectPipeline: - """ - Pipeline for decrypting objects after they are retrieved from S3. + """Pipeline for decrypting objects after they are retrieved from S3. This pipeline handles only the decryption process for S3 objects. The actual S3 API calls are handled by the S3EncryptionClient. @@ -90,8 +86,7 @@ class GetEncryptedObjectPipeline: cmm: AbstractCryptoMaterialsManager = field() def decrypt(self, response, encryption_context={}): - """ - Decrypt the data after it is retrieved from S3. + """Decrypt the data after it is retrieved from S3. Args: response (dict): The response from S3 containing the encrypted data and metadata diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index d956f321..16446b1d 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -4,7 +4,6 @@ from datetime import datetime import boto3 - from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.materials.kms_keyring import KmsKeyring diff --git a/test/test_encryption_materials_integration.py b/test/test_encryption_materials_integration.py index da8a17b2..be16662d 100644 --- a/test/test_encryption_materials_integration.py +++ b/test/test_encryption_materials_integration.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock from src.s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager from src.s3_encryption.materials.encrypted_data_key import EncryptedDataKey diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..e2aaa475 --- /dev/null +++ b/uv.lock @@ -0,0 +1,380 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "amazon-s3-encryption-client-python" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "attrs" }, + { name = "boto3" }, + { name = "cryptography" }, +] + +[package.optional-dependencies] +dev = [ + { name = "black" }, + { name = "isort" }, + { name = "ruff" }, +] +test = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "attrs", specifier = ">=25.1.0" }, + { name = "black", marker = "extra == 'dev'", specifier = ">=24.3.0" }, + { name = "boto3", specifier = ">=1.37.2" }, + { name = "cryptography", specifier = ">=45.0.6" }, + { name = "isort", marker = "extra == 'dev'", specifier = ">=5.13.2" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8.4.1" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.3.0" }, +] +provides-extras = ["test", "dev"] + +[[package]] +name = "attrs" +version = "25.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/1367933a8532ee6ff8d63537de4f1177af4bff9f3e829baf7331f595bb24/attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b", size = 812032, upload-time = "2025-03-13T11:10:22.779Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, +] + +[[package]] +name = "black" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "mypy-extensions" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "platformdirs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/94/49/26a7b0f3f35da4b5a65f081943b7bcd22d7002f5f0fb8098ec1ff21cb6ef/black-25.1.0.tar.gz", hash = "sha256:33496d5cd1222ad73391352b4ae8da15253c5de89b93a80b3e2c8d9a19ec2666", size = 649449, upload-time = "2025-01-29T04:15:40.373Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/4f/87f596aca05c3ce5b94b8663dbfe242a12843caaa82dd3f85f1ffdc3f177/black-25.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a39337598244de4bae26475f77dda852ea00a93bd4c728e09eacd827ec929df0", size = 1614372, upload-time = "2025-01-29T05:37:11.71Z" }, + { url = "https://files.pythonhosted.org/packages/e7/d0/2c34c36190b741c59c901e56ab7f6e54dad8df05a6272a9747ecef7c6036/black-25.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:96c1c7cd856bba8e20094e36e0f948718dc688dba4a9d78c3adde52b9e6c2299", size = 1442865, upload-time = "2025-01-29T05:37:14.309Z" }, + { url = "https://files.pythonhosted.org/packages/21/d4/7518c72262468430ead45cf22bd86c883a6448b9eb43672765d69a8f1248/black-25.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bce2e264d59c91e52d8000d507eb20a9aca4a778731a08cfff7e5ac4a4bb7096", size = 1749699, upload-time = "2025-01-29T04:18:17.688Z" }, + { url = "https://files.pythonhosted.org/packages/58/db/4f5beb989b547f79096e035c4981ceb36ac2b552d0ac5f2620e941501c99/black-25.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:172b1dbff09f86ce6f4eb8edf9dede08b1fce58ba194c87d7a4f1a5aa2f5b3c2", size = 1428028, upload-time = "2025-01-29T04:18:51.711Z" }, + { url = "https://files.pythonhosted.org/packages/83/71/3fe4741df7adf015ad8dfa082dd36c94ca86bb21f25608eb247b4afb15b2/black-25.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4b60580e829091e6f9238c848ea6750efed72140b91b048770b64e74fe04908b", size = 1650988, upload-time = "2025-01-29T05:37:16.707Z" }, + { url = "https://files.pythonhosted.org/packages/13/f3/89aac8a83d73937ccd39bbe8fc6ac8860c11cfa0af5b1c96d081facac844/black-25.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e2978f6df243b155ef5fa7e558a43037c3079093ed5d10fd84c43900f2d8ecc", size = 1453985, upload-time = "2025-01-29T05:37:18.273Z" }, + { url = "https://files.pythonhosted.org/packages/6f/22/b99efca33f1f3a1d2552c714b1e1b5ae92efac6c43e790ad539a163d1754/black-25.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b48735872ec535027d979e8dcb20bf4f70b5ac75a8ea99f127c106a7d7aba9f", size = 1783816, upload-time = "2025-01-29T04:18:33.823Z" }, + { url = "https://files.pythonhosted.org/packages/18/7e/a27c3ad3822b6f2e0e00d63d58ff6299a99a5b3aee69fa77cd4b0076b261/black-25.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:ea0213189960bda9cf99be5b8c8ce66bb054af5e9e861249cd23471bd7b0b3ba", size = 1440860, upload-time = "2025-01-29T04:19:12.944Z" }, + { url = "https://files.pythonhosted.org/packages/98/87/0edf98916640efa5d0696e1abb0a8357b52e69e82322628f25bf14d263d1/black-25.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f0b18a02996a836cc9c9c78e5babec10930862827b1b724ddfe98ccf2f2fe4f", size = 1650673, upload-time = "2025-01-29T05:37:20.574Z" }, + { url = "https://files.pythonhosted.org/packages/52/e5/f7bf17207cf87fa6e9b676576749c6b6ed0d70f179a3d812c997870291c3/black-25.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:afebb7098bfbc70037a053b91ae8437c3857482d3a690fefc03e9ff7aa9a5fd3", size = 1453190, upload-time = "2025-01-29T05:37:22.106Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ee/adda3d46d4a9120772fae6de454c8495603c37c4c3b9c60f25b1ab6401fe/black-25.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:030b9759066a4ee5e5aca28c3c77f9c64789cdd4de8ac1df642c40b708be6171", size = 1782926, upload-time = "2025-01-29T04:18:58.564Z" }, + { url = "https://files.pythonhosted.org/packages/cc/64/94eb5f45dcb997d2082f097a3944cfc7fe87e071907f677e80788a2d7b7a/black-25.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:a22f402b410566e2d1c950708c77ebf5ebd5d0d88a6a2e87c86d9fb48afa0d18", size = 1442613, upload-time = "2025-01-29T04:19:27.63Z" }, + { url = "https://files.pythonhosted.org/packages/09/71/54e999902aed72baf26bca0d50781b01838251a462612966e9fc4891eadd/black-25.1.0-py3-none-any.whl", hash = "sha256:95e8176dae143ba9097f351d174fdaf0ccd29efb414b362ae3fd72bf0f710717", size = 207646, upload-time = "2025-01-29T04:15:38.082Z" }, +] + +[[package]] +name = "boto3" +version = "1.40.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/5a/f31556d817e872c2723196a34b197d971d78297b22b8bae0ae6d93f7f9c1/boto3-1.40.7.tar.gz", hash = "sha256:61b15f70761f1eadd721c6ba41a92658f003eaaef09500ca7642f5ae68ec8945", size = 111989, upload-time = "2025-08-11T19:20:45.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/e3/f2a77f4809ffe4e896c2e6186db88333ae980f52a91b28e9fd068d8f5506/boto3-1.40.7-py3-none-any.whl", hash = "sha256:8727cac601a679d2885dc78b8119a0548bbbe04e49b72f7d94021a629154c080", size = 140061, upload-time = "2025-08-11T19:20:43.173Z" }, +] + +[[package]] +name = "botocore" +version = "1.40.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/d7/5e559918410b259c1e54a4646ff39c56433e1c9cefa5e66ab0f06716cee8/botocore-1.40.7.tar.gz", hash = "sha256:33793696680cf3a0c4b5ace4f9070c67c4d4fcb19c999fd85cfee55de3dcf913", size = 14318282, upload-time = "2025-08-11T19:20:33.348Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/fa/bb7ec68b24d1b4678d341a305cbfed78a593e6383c86a70727410e4d0e11/botocore-1.40.7-py3-none-any.whl", hash = "sha256:a06956f3d7222e80ef6ae193608f358c3b7898e1a2b88553479d8f9737fbb03e", size = 13981488, upload-time = "2025-08-11T19:20:27.303Z" }, +] + +[[package]] +name = "cffi" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload-time = "2024-09-04T20:45:21.852Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264, upload-time = "2024-09-04T20:43:51.124Z" }, + { url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651, upload-time = "2024-09-04T20:43:52.872Z" }, + { url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259, upload-time = "2024-09-04T20:43:56.123Z" }, + { url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200, upload-time = "2024-09-04T20:43:57.891Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235, upload-time = "2024-09-04T20:44:00.18Z" }, + { url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721, upload-time = "2024-09-04T20:44:01.585Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242, upload-time = "2024-09-04T20:44:03.467Z" }, + { url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999, upload-time = "2024-09-04T20:44:05.023Z" }, + { url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242, upload-time = "2024-09-04T20:44:06.444Z" }, + { url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604, upload-time = "2024-09-04T20:44:08.206Z" }, + { url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727, upload-time = "2024-09-04T20:44:09.481Z" }, + { url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400, upload-time = "2024-09-04T20:44:10.873Z" }, + { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178, upload-time = "2024-09-04T20:44:12.232Z" }, + { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840, upload-time = "2024-09-04T20:44:13.739Z" }, + { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803, upload-time = "2024-09-04T20:44:15.231Z" }, + { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850, upload-time = "2024-09-04T20:44:17.188Z" }, + { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729, upload-time = "2024-09-04T20:44:18.688Z" }, + { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256, upload-time = "2024-09-04T20:44:20.248Z" }, + { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424, upload-time = "2024-09-04T20:44:21.673Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568, upload-time = "2024-09-04T20:44:23.245Z" }, + { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736, upload-time = "2024-09-04T20:44:24.757Z" }, + { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448, upload-time = "2024-09-04T20:44:26.208Z" }, + { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976, upload-time = "2024-09-04T20:44:27.578Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989, upload-time = "2024-09-04T20:44:28.956Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802, upload-time = "2024-09-04T20:44:30.289Z" }, + { url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792, upload-time = "2024-09-04T20:44:32.01Z" }, + { url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893, upload-time = "2024-09-04T20:44:33.606Z" }, + { url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810, upload-time = "2024-09-04T20:44:35.191Z" }, + { url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200, upload-time = "2024-09-04T20:44:36.743Z" }, + { url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447, upload-time = "2024-09-04T20:44:38.492Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358, upload-time = "2024-09-04T20:44:40.046Z" }, + { url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469, upload-time = "2024-09-04T20:44:41.616Z" }, + { url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475, upload-time = "2024-09-04T20:44:43.733Z" }, + { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009, upload-time = "2024-09-04T20:44:45.309Z" }, +] + +[[package]] +name = "click" +version = "8.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/60/6c/8ca2efa64cf75a977a0d7fac081354553ebe483345c734fb6b6515d96bbc/click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202", size = 286342, upload-time = "2025-05-20T23:19:49.832Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215, upload-time = "2025-05-20T23:19:47.796Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "cryptography" +version = "45.0.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d6/0d/d13399c94234ee8f3df384819dc67e0c5ce215fb751d567a55a1f4b028c7/cryptography-45.0.6.tar.gz", hash = "sha256:5c966c732cf6e4a276ce83b6e4c729edda2df6929083a952cc7da973c539c719", size = 744949, upload-time = "2025-08-05T23:59:27.93Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/29/2793d178d0eda1ca4a09a7c4e09a5185e75738cc6d526433e8663b460ea6/cryptography-45.0.6-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:048e7ad9e08cf4c0ab07ff7f36cc3115924e22e2266e034450a890d9e312dd74", size = 7042702, upload-time = "2025-08-05T23:58:23.464Z" }, + { url = "https://files.pythonhosted.org/packages/b3/b6/cabd07410f222f32c8d55486c464f432808abaa1f12af9afcbe8f2f19030/cryptography-45.0.6-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:44647c5d796f5fc042bbc6d61307d04bf29bccb74d188f18051b635f20a9c75f", size = 4206483, upload-time = "2025-08-05T23:58:27.132Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9e/f9c7d36a38b1cfeb1cc74849aabe9bf817990f7603ff6eb485e0d70e0b27/cryptography-45.0.6-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e40b80ecf35ec265c452eea0ba94c9587ca763e739b8e559c128d23bff7ebbbf", size = 4429679, upload-time = "2025-08-05T23:58:29.152Z" }, + { url = "https://files.pythonhosted.org/packages/9c/2a/4434c17eb32ef30b254b9e8b9830cee4e516f08b47fdd291c5b1255b8101/cryptography-45.0.6-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:00e8724bdad672d75e6f069b27970883179bd472cd24a63f6e620ca7e41cc0c5", size = 4210553, upload-time = "2025-08-05T23:58:30.596Z" }, + { url = "https://files.pythonhosted.org/packages/ef/1d/09a5df8e0c4b7970f5d1f3aff1b640df6d4be28a64cae970d56c6cf1c772/cryptography-45.0.6-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7a3085d1b319d35296176af31c90338eeb2ddac8104661df79f80e1d9787b8b2", size = 3894499, upload-time = "2025-08-05T23:58:32.03Z" }, + { url = "https://files.pythonhosted.org/packages/79/62/120842ab20d9150a9d3a6bdc07fe2870384e82f5266d41c53b08a3a96b34/cryptography-45.0.6-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1b7fa6a1c1188c7ee32e47590d16a5a0646270921f8020efc9a511648e1b2e08", size = 4458484, upload-time = "2025-08-05T23:58:33.526Z" }, + { url = "https://files.pythonhosted.org/packages/fd/80/1bc3634d45ddfed0871bfba52cf8f1ad724761662a0c792b97a951fb1b30/cryptography-45.0.6-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:275ba5cc0d9e320cd70f8e7b96d9e59903c815ca579ab96c1e37278d231fc402", size = 4210281, upload-time = "2025-08-05T23:58:35.445Z" }, + { url = "https://files.pythonhosted.org/packages/7d/fe/ffb12c2d83d0ee625f124880a1f023b5878f79da92e64c37962bbbe35f3f/cryptography-45.0.6-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:f4028f29a9f38a2025abedb2e409973709c660d44319c61762202206ed577c42", size = 4456890, upload-time = "2025-08-05T23:58:36.923Z" }, + { url = "https://files.pythonhosted.org/packages/8c/8e/b3f3fe0dc82c77a0deb5f493b23311e09193f2268b77196ec0f7a36e3f3e/cryptography-45.0.6-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ee411a1b977f40bd075392c80c10b58025ee5c6b47a822a33c1198598a7a5f05", size = 4333247, upload-time = "2025-08-05T23:58:38.781Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a6/c3ef2ab9e334da27a1d7b56af4a2417d77e7806b2e0f90d6267ce120d2e4/cryptography-45.0.6-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e2a21a8eda2d86bb604934b6b37691585bd095c1f788530c1fcefc53a82b3453", size = 4565045, upload-time = "2025-08-05T23:58:40.415Z" }, + { url = "https://files.pythonhosted.org/packages/31/c3/77722446b13fa71dddd820a5faab4ce6db49e7e0bf8312ef4192a3f78e2f/cryptography-45.0.6-cp311-abi3-win32.whl", hash = "sha256:d063341378d7ee9c91f9d23b431a3502fc8bfacd54ef0a27baa72a0843b29159", size = 2928923, upload-time = "2025-08-05T23:58:41.919Z" }, + { url = "https://files.pythonhosted.org/packages/38/63/a025c3225188a811b82932a4dcc8457a26c3729d81578ccecbcce2cb784e/cryptography-45.0.6-cp311-abi3-win_amd64.whl", hash = "sha256:833dc32dfc1e39b7376a87b9a6a4288a10aae234631268486558920029b086ec", size = 3403805, upload-time = "2025-08-05T23:58:43.792Z" }, + { url = "https://files.pythonhosted.org/packages/5b/af/bcfbea93a30809f126d51c074ee0fac5bd9d57d068edf56c2a73abedbea4/cryptography-45.0.6-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:3436128a60a5e5490603ab2adbabc8763613f638513ffa7d311c900a8349a2a0", size = 7020111, upload-time = "2025-08-05T23:58:45.316Z" }, + { url = "https://files.pythonhosted.org/packages/98/c6/ea5173689e014f1a8470899cd5beeb358e22bb3cf5a876060f9d1ca78af4/cryptography-45.0.6-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0d9ef57b6768d9fa58e92f4947cea96ade1233c0e236db22ba44748ffedca394", size = 4198169, upload-time = "2025-08-05T23:58:47.121Z" }, + { url = "https://files.pythonhosted.org/packages/ba/73/b12995edc0c7e2311ffb57ebd3b351f6b268fed37d93bfc6f9856e01c473/cryptography-45.0.6-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea3c42f2016a5bbf71825537c2ad753f2870191134933196bee408aac397b3d9", size = 4421273, upload-time = "2025-08-05T23:58:48.557Z" }, + { url = "https://files.pythonhosted.org/packages/f7/6e/286894f6f71926bc0da67408c853dd9ba953f662dcb70993a59fd499f111/cryptography-45.0.6-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:20ae4906a13716139d6d762ceb3e0e7e110f7955f3bc3876e3a07f5daadec5f3", size = 4199211, upload-time = "2025-08-05T23:58:50.139Z" }, + { url = "https://files.pythonhosted.org/packages/de/34/a7f55e39b9623c5cb571d77a6a90387fe557908ffc44f6872f26ca8ae270/cryptography-45.0.6-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dac5ec199038b8e131365e2324c03d20e97fe214af051d20c49db129844e8b3", size = 3883732, upload-time = "2025-08-05T23:58:52.253Z" }, + { url = "https://files.pythonhosted.org/packages/f9/b9/c6d32edbcba0cd9f5df90f29ed46a65c4631c4fbe11187feb9169c6ff506/cryptography-45.0.6-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:18f878a34b90d688982e43f4b700408b478102dd58b3e39de21b5ebf6509c301", size = 4450655, upload-time = "2025-08-05T23:58:53.848Z" }, + { url = "https://files.pythonhosted.org/packages/77/2d/09b097adfdee0227cfd4c699b3375a842080f065bab9014248933497c3f9/cryptography-45.0.6-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5bd6020c80c5b2b2242d6c48487d7b85700f5e0038e67b29d706f98440d66eb5", size = 4198956, upload-time = "2025-08-05T23:58:55.209Z" }, + { url = "https://files.pythonhosted.org/packages/55/66/061ec6689207d54effdff535bbdf85cc380d32dd5377173085812565cf38/cryptography-45.0.6-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:eccddbd986e43014263eda489abbddfbc287af5cddfd690477993dbb31e31016", size = 4449859, upload-time = "2025-08-05T23:58:56.639Z" }, + { url = "https://files.pythonhosted.org/packages/41/ff/e7d5a2ad2d035e5a2af116e1a3adb4d8fcd0be92a18032917a089c6e5028/cryptography-45.0.6-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:550ae02148206beb722cfe4ef0933f9352bab26b087af00e48fdfb9ade35c5b3", size = 4320254, upload-time = "2025-08-05T23:58:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/82/27/092d311af22095d288f4db89fcaebadfb2f28944f3d790a4cf51fe5ddaeb/cryptography-45.0.6-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5b64e668fc3528e77efa51ca70fadcd6610e8ab231e3e06ae2bab3b31c2b8ed9", size = 4554815, upload-time = "2025-08-05T23:59:00.283Z" }, + { url = "https://files.pythonhosted.org/packages/7e/01/aa2f4940262d588a8fdf4edabe4cda45854d00ebc6eaac12568b3a491a16/cryptography-45.0.6-cp37-abi3-win32.whl", hash = "sha256:780c40fb751c7d2b0c6786ceee6b6f871e86e8718a8ff4bc35073ac353c7cd02", size = 2912147, upload-time = "2025-08-05T23:59:01.716Z" }, + { url = "https://files.pythonhosted.org/packages/0a/bc/16e0276078c2de3ceef6b5a34b965f4436215efac45313df90d55f0ba2d2/cryptography-45.0.6-cp37-abi3-win_amd64.whl", hash = "sha256:20d15aed3ee522faac1a39fbfdfee25d17b1284bafd808e1640a74846d7c4d1b", size = 3390459, upload-time = "2025-08-05T23:59:03.358Z" }, + { url = "https://files.pythonhosted.org/packages/61/69/c252de4ec047ba2f567ecb53149410219577d408c2aea9c989acae7eafce/cryptography-45.0.6-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:fc022c1fa5acff6def2fc6d7819bbbd31ccddfe67d075331a65d9cfb28a20983", size = 3584669, upload-time = "2025-08-05T23:59:15.431Z" }, + { url = "https://files.pythonhosted.org/packages/e3/fe/deea71e9f310a31fe0a6bfee670955152128d309ea2d1c79e2a5ae0f0401/cryptography-45.0.6-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:3de77e4df42ac8d4e4d6cdb342d989803ad37707cf8f3fbf7b088c9cbdd46427", size = 4153022, upload-time = "2025-08-05T23:59:16.954Z" }, + { url = "https://files.pythonhosted.org/packages/60/45/a77452f5e49cb580feedba6606d66ae7b82c128947aa754533b3d1bd44b0/cryptography-45.0.6-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:599c8d7df950aa68baa7e98f7b73f4f414c9f02d0e8104a30c0182a07732638b", size = 4386802, upload-time = "2025-08-05T23:59:18.55Z" }, + { url = "https://files.pythonhosted.org/packages/a3/b9/a2f747d2acd5e3075fdf5c145c7c3568895daaa38b3b0c960ef830db6cdc/cryptography-45.0.6-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:31a2b9a10530a1cb04ffd6aa1cd4d3be9ed49f7d77a4dafe198f3b382f41545c", size = 4152706, upload-time = "2025-08-05T23:59:20.044Z" }, + { url = "https://files.pythonhosted.org/packages/81/ec/381b3e8d0685a3f3f304a382aa3dfce36af2d76467da0fd4bb21ddccc7b2/cryptography-45.0.6-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:e5b3dda1b00fb41da3af4c5ef3f922a200e33ee5ba0f0bc9ecf0b0c173958385", size = 4386740, upload-time = "2025-08-05T23:59:21.525Z" }, + { url = "https://files.pythonhosted.org/packages/0a/76/cf8d69da8d0b5ecb0db406f24a63a3f69ba5e791a11b782aeeefef27ccbb/cryptography-45.0.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:629127cfdcdc6806dfe234734d7cb8ac54edaf572148274fa377a7d3405b0043", size = 3331874, upload-time = "2025-08-05T23:59:23.017Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, +] + +[[package]] +name = "isort" +version = "6.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b8/21/1e2a441f74a653a144224d7d21afe8f4169e6c7c20bb13aec3a2dc3815e0/isort-6.0.1.tar.gz", hash = "sha256:1cb5df28dfbc742e490c5e41bad6da41b805b0a8be7bc93cd0fb2a8a890ac450", size = 821955, upload-time = "2025-02-26T21:13:16.955Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/11/114d0a5f4dabbdcedc1125dee0888514c3c3b16d3e9facad87ed96fad97c/isort-6.0.1-py3-none-any.whl", hash = "sha256:2dc5d7f65c9678d94c88dfc29161a320eec67328bc97aad576874cb4be1e9615", size = 94186, upload-time = "2025-02-26T21:13:14.911Z" }, +] + +[[package]] +name = "jmespath" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.3.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/8b/3c73abc9c759ecd3f1f7ceff6685840859e8070c4d947c93fae71f6a0bf2/platformdirs-4.3.8.tar.gz", hash = "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc", size = 21362, upload-time = "2025-05-07T22:47:42.121Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/39/979e8e21520d4e47a0bbe349e2713c0aac6f3d853d0e5b34d76206c439aa/platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4", size = 18567, upload-time = "2025-05-07T22:47:40.376Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pycparser" +version = "2.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736, upload-time = "2024-03-30T13:22:22.564Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552, upload-time = "2024-03-30T13:22:20.476Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/08/ba/45911d754e8eba3d5a841a5ce61a65a685ff1798421ac054f85aa8747dfb/pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c", size = 1517714, upload-time = "2025-06-18T05:48:06.109Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474, upload-time = "2025-06-18T05:48:03.955Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "ruff" +version = "0.12.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4b/da/5bd7565be729e86e1442dad2c9a364ceeff82227c2dece7c29697a9795eb/ruff-0.12.8.tar.gz", hash = "sha256:4cb3a45525176e1009b2b64126acf5f9444ea59066262791febf55e40493a033", size = 5242373, upload-time = "2025-08-07T19:05:47.268Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/1e/c843bfa8ad1114fab3eb2b78235dda76acd66384c663a4e0415ecc13aa1e/ruff-0.12.8-py3-none-linux_armv6l.whl", hash = "sha256:63cb5a5e933fc913e5823a0dfdc3c99add73f52d139d6cd5cc8639d0e0465513", size = 11675315, upload-time = "2025-08-07T19:05:06.15Z" }, + { url = "https://files.pythonhosted.org/packages/24/ee/af6e5c2a8ca3a81676d5480a1025494fd104b8896266502bb4de2a0e8388/ruff-0.12.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9a9bbe28f9f551accf84a24c366c1aa8774d6748438b47174f8e8565ab9dedbc", size = 12456653, upload-time = "2025-08-07T19:05:09.759Z" }, + { url = "https://files.pythonhosted.org/packages/99/9d/e91f84dfe3866fa648c10512904991ecc326fd0b66578b324ee6ecb8f725/ruff-0.12.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:2fae54e752a3150f7ee0e09bce2e133caf10ce9d971510a9b925392dc98d2fec", size = 11659690, upload-time = "2025-08-07T19:05:12.551Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ac/a363d25ec53040408ebdd4efcee929d48547665858ede0505d1d8041b2e5/ruff-0.12.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0acbcf01206df963d9331b5838fb31f3b44fa979ee7fa368b9b9057d89f4a53", size = 11896923, upload-time = "2025-08-07T19:05:14.821Z" }, + { url = "https://files.pythonhosted.org/packages/58/9f/ea356cd87c395f6ade9bb81365bd909ff60860975ca1bc39f0e59de3da37/ruff-0.12.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ae3e7504666ad4c62f9ac8eedb52a93f9ebdeb34742b8b71cd3cccd24912719f", size = 11477612, upload-time = "2025-08-07T19:05:16.712Z" }, + { url = "https://files.pythonhosted.org/packages/1a/46/92e8fa3c9dcfd49175225c09053916cb97bb7204f9f899c2f2baca69e450/ruff-0.12.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb82efb5d35d07497813a1c5647867390a7d83304562607f3579602fa3d7d46f", size = 13182745, upload-time = "2025-08-07T19:05:18.709Z" }, + { url = "https://files.pythonhosted.org/packages/5e/c4/f2176a310f26e6160deaf661ef60db6c3bb62b7a35e57ae28f27a09a7d63/ruff-0.12.8-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:dbea798fc0065ad0b84a2947b0aff4233f0cb30f226f00a2c5850ca4393de609", size = 14206885, upload-time = "2025-08-07T19:05:21.025Z" }, + { url = "https://files.pythonhosted.org/packages/87/9d/98e162f3eeeb6689acbedbae5050b4b3220754554526c50c292b611d3a63/ruff-0.12.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:49ebcaccc2bdad86fd51b7864e3d808aad404aab8df33d469b6e65584656263a", size = 13639381, upload-time = "2025-08-07T19:05:23.423Z" }, + { url = "https://files.pythonhosted.org/packages/81/4e/1b7478b072fcde5161b48f64774d6edd59d6d198e4ba8918d9f4702b8043/ruff-0.12.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ac9c570634b98c71c88cb17badd90f13fc076a472ba6ef1d113d8ed3df109fb", size = 12613271, upload-time = "2025-08-07T19:05:25.507Z" }, + { url = "https://files.pythonhosted.org/packages/e8/67/0c3c9179a3ad19791ef1b8f7138aa27d4578c78700551c60d9260b2c660d/ruff-0.12.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:560e0cd641e45591a3e42cb50ef61ce07162b9c233786663fdce2d8557d99818", size = 12847783, upload-time = "2025-08-07T19:05:28.14Z" }, + { url = "https://files.pythonhosted.org/packages/4e/2a/0b6ac3dd045acf8aa229b12c9c17bb35508191b71a14904baf99573a21bd/ruff-0.12.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:71c83121512e7743fba5a8848c261dcc454cafb3ef2934a43f1b7a4eb5a447ea", size = 11702672, upload-time = "2025-08-07T19:05:30.413Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ee/f9fdc9f341b0430110de8b39a6ee5fa68c5706dc7c0aa940817947d6937e/ruff-0.12.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:de4429ef2ba091ecddedd300f4c3f24bca875d3d8b23340728c3cb0da81072c3", size = 11440626, upload-time = "2025-08-07T19:05:32.492Z" }, + { url = "https://files.pythonhosted.org/packages/89/fb/b3aa2d482d05f44e4d197d1de5e3863feb13067b22c571b9561085c999dc/ruff-0.12.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a2cab5f60d5b65b50fba39a8950c8746df1627d54ba1197f970763917184b161", size = 12462162, upload-time = "2025-08-07T19:05:34.449Z" }, + { url = "https://files.pythonhosted.org/packages/18/9f/5c5d93e1d00d854d5013c96e1a92c33b703a0332707a7cdbd0a4880a84fb/ruff-0.12.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:45c32487e14f60b88aad6be9fd5da5093dbefb0e3e1224131cb1d441d7cb7d46", size = 12913212, upload-time = "2025-08-07T19:05:36.541Z" }, + { url = "https://files.pythonhosted.org/packages/71/13/ab9120add1c0e4604c71bfc2e4ef7d63bebece0cfe617013da289539cef8/ruff-0.12.8-py3-none-win32.whl", hash = "sha256:daf3475060a617fd5bc80638aeaf2f5937f10af3ec44464e280a9d2218e720d3", size = 11694382, upload-time = "2025-08-07T19:05:38.468Z" }, + { url = "https://files.pythonhosted.org/packages/f6/dc/a2873b7c5001c62f46266685863bee2888caf469d1edac84bf3242074be2/ruff-0.12.8-py3-none-win_amd64.whl", hash = "sha256:7209531f1a1fcfbe8e46bcd7ab30e2f43604d8ba1c49029bb420b103d0b5f76e", size = 12740482, upload-time = "2025-08-07T19:05:40.391Z" }, + { url = "https://files.pythonhosted.org/packages/cb/5c/799a1efb8b5abab56e8a9f2a0b72d12bd64bb55815e9476c7d0a2887d2f7/ruff-0.12.8-py3-none-win_arm64.whl", hash = "sha256:c90e1a334683ce41b0e7a04f41790c429bf5073b62c1ae701c9dc5b3d14f0749", size = 11884718, upload-time = "2025-08-07T19:05:42.866Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.13.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/05/d52bf1e65044b4e5e27d4e63e8d1579dbdec54fce685908ae09bc3720030/s3transfer-0.13.1.tar.gz", hash = "sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf", size = 150589, upload-time = "2025-07-18T19:22:42.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/4f/d073e09df851cfa251ef7840007d04db3293a0482ce607d2b993926089be/s3transfer-0.13.1-py3-none-any.whl", hash = "sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724", size = 85308, upload-time = "2025-07-18T19:22:40.947Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "urllib3" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, +] From 351431a780ff088d6054b8bcf2fee74aa5e7bfff Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 11 Aug 2025 13:21:53 -0700 Subject: [PATCH 16/81] GHA stuff --- .github/workflows/lint.yml | 36 ++++++++++++++++++++++++++++++++++++ .github/workflows/main.yml | 4 ++++ .github/workflows/test.yml | 14 +++++--------- 3 files changed, 45 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..3f5d091a --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,36 @@ +name: Lint + +on: + push: + branches: [ main ] + pull_request: + workflow_dispatch: + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install Uv + run: pip install uv + + - name: Install dependencies + run: uv pip install -e ".[dev,test]" + + - name: Run Black + run: uv run black --check . + + - name: Run isort + run: uv run isort --check . + + - name: Run Ruff + # Allow ruff to fail for now as we're gradually adopting linting standards + run: uv run ruff check src/ test/ || true diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b7396de4..e10b7d0d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,6 +13,10 @@ on: type: string jobs: + lint: + name: Lint + uses: ./.github/workflows/lint.yml + run-tests: name: Run Tests uses: ./.github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c60d4a20..68a0d1e1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,15 +26,11 @@ jobs: with: python-version: ${{ inputs.python-version || '3.11' }} - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - version: 1.7.1 - virtualenvs-create: true - virtualenvs-in-project: true + - name: Install Uv + run: pip install uv - name: Install dependencies - run: poetry install + run: uv pip install -e ".[dev,test]" - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 @@ -43,10 +39,10 @@ jobs: aws-region: us-west-2 - name: Run unit tests - run: poetry run pytest test/ --verbose + run: uv run pytest test/ --verbose - name: Run integration tests - run: poetry run pytest test/integration/ --verbose + run: uv run pytest test/integration/ --verbose env: CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} From c15f2da80e9efec778e6a8ee94cda2b9deb5f9f2 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 11 Aug 2025 13:25:09 -0700 Subject: [PATCH 17/81] uv venv --- .github/workflows/lint.yml | 3 +++ .github/workflows/test.yml | 3 +++ Makefile | 1 + 3 files changed, 7 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3f5d091a..f4cd3512 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -22,6 +22,9 @@ jobs: - name: Install Uv run: pip install uv + - name: Create virtual environment + run: uv venv + - name: Install dependencies run: uv pip install -e ".[dev,test]" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 68a0d1e1..8b07e40a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,6 +29,9 @@ jobs: - name: Install Uv run: pip install uv + - name: Create virtual environment + run: uv venv + - name: Install dependencies run: uv pip install -e ".[dev,test]" diff --git a/Makefile b/Makefile index 65578384..fb1a83b0 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ all: lint test # Install dependencies install: + uv venv uv pip install -e ".[dev,test]" # Run linting checks From 5e3f122224c7c796051011ff2832a0aca5d24b18 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 11 Aug 2025 13:32:49 -0700 Subject: [PATCH 18/81] black --- src/s3_encryption/__init__.py | 3 +-- src/s3_encryption/materials/keyring.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index fb633945..c19ca28d 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -16,8 +16,7 @@ @define class S3EncryptionClientConfig: - """Configuration object for the S3 Encryption Client - """ + """Configuration object for the S3 Encryption Client""" keyring: AbstractKeyring cmm: AbstractCryptoMaterialsManager = field() diff --git a/src/s3_encryption/materials/keyring.py b/src/s3_encryption/materials/keyring.py index f840bce4..aed20e96 100644 --- a/src/s3_encryption/materials/keyring.py +++ b/src/s3_encryption/materials/keyring.py @@ -42,8 +42,7 @@ def onDecrypt(self, decMaterials, encrypted_data_keys=None): @define class S3Keyring(AbstractKeyring): - """Base class for S3 encryption keyrings that provides common validation logic. - """ + """Base class for S3 encryption keyrings that provides common validation logic.""" # Ideally this would be set, but attrs doesn't play nice # enable_legacy_wrapping_algorithms: bool = field(default=False) From dac7ab7c0d6a47ab1e4574bb1d6a919af4f5e1aa Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 11 Aug 2025 13:47:02 -0700 Subject: [PATCH 19/81] only ruff, remove isort --- Makefile | 4 +--- pyproject.toml | 7 ++----- test/integration/test_i_s3_encryption.py | 1 + uv.lock | 11 ----------- 4 files changed, 4 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index fb1a83b0..80baca1d 100644 --- a/Makefile +++ b/Makefile @@ -11,14 +11,12 @@ install: # Run linting checks lint: uv run black --check . - uv run isort --check . # Allow ruff to fail for now as we're gradually adopting linting standards uv run ruff check src/ test/ || true -# Format code with Black, isort, and Ruff +# Format code with Black and Ruff format: uv run black . - uv run isort . uv run ruff check --fix src/ test/ # Run all tests diff --git a/pyproject.toml b/pyproject.toml index a613bc12..883b6914 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,6 @@ test = [ dev = [ "black>=24.3.0", "ruff>=0.3.0", - "isort>=5.13.2", ] [build-system] @@ -51,7 +50,5 @@ convention = "google" [tool.ruff.lint.mccabe] max-complexity = 10 -[tool.isort] -profile = "black" -line_length = 100 -known_first_party = ["s3_encryption"] +[tool.ruff.lint.isort] +known-first-party = ["s3_encryption"] diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index 16446b1d..d956f321 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -4,6 +4,7 @@ from datetime import datetime import boto3 + from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.materials.kms_keyring import KmsKeyring diff --git a/uv.lock b/uv.lock index e2aaa475..ebc7a7e9 100644 --- a/uv.lock +++ b/uv.lock @@ -15,7 +15,6 @@ dependencies = [ [package.optional-dependencies] dev = [ { name = "black" }, - { name = "isort" }, { name = "ruff" }, ] test = [ @@ -28,7 +27,6 @@ requires-dist = [ { name = "black", marker = "extra == 'dev'", specifier = ">=24.3.0" }, { name = "boto3", specifier = ">=1.37.2" }, { name = "cryptography", specifier = ">=45.0.6" }, - { name = "isort", marker = "extra == 'dev'", specifier = ">=5.13.2" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.4.1" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.3.0" }, ] @@ -215,15 +213,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, ] -[[package]] -name = "isort" -version = "6.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b8/21/1e2a441f74a653a144224d7d21afe8f4169e6c7c20bb13aec3a2dc3815e0/isort-6.0.1.tar.gz", hash = "sha256:1cb5df28dfbc742e490c5e41bad6da41b805b0a8be7bc93cd0fb2a8a890ac450", size = 821955, upload-time = "2025-02-26T21:13:16.955Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/11/114d0a5f4dabbdcedc1125dee0888514c3c3b16d3e9facad87ed96fad97c/isort-6.0.1-py3-none-any.whl", hash = "sha256:2dc5d7f65c9678d94c88dfc29161a320eec67328bc97aad576874cb4be1e9615", size = 94186, upload-time = "2025-02-26T21:13:14.911Z" }, -] - [[package]] name = "jmespath" version = "1.0.1" From 3cd450bc552b516d9b8e3b338706196e79227a76 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 11 Aug 2025 13:51:04 -0700 Subject: [PATCH 20/81] use makefile --- .github/workflows/lint.yml | 19 ++++--------------- .github/workflows/test.yml | 9 +++------ 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f4cd3512..8fc3bdc7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -22,18 +22,7 @@ jobs: - name: Install Uv run: pip install uv - - name: Create virtual environment - run: uv venv - - - name: Install dependencies - run: uv pip install -e ".[dev,test]" - - - name: Run Black - run: uv run black --check . - - - name: Run isort - run: uv run isort --check . - - - name: Run Ruff - # Allow ruff to fail for now as we're gradually adopting linting standards - run: uv run ruff check src/ test/ || true + - name: Install dependencies and run linting + run: | + make install + make lint diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8b07e40a..0bdc88de 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,11 +29,8 @@ jobs: - name: Install Uv run: pip install uv - - name: Create virtual environment - run: uv venv - - name: Install dependencies - run: uv pip install -e ".[dev,test]" + run: make install - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 @@ -42,10 +39,10 @@ jobs: aws-region: us-west-2 - name: Run unit tests - run: uv run pytest test/ --verbose + run: make test-unit - name: Run integration tests - run: uv run pytest test/integration/ --verbose + run: make test-integration env: CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} From be51b702cb4ce0c7ff41e69759757e655d7ce2b0 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 11 Aug 2025 13:57:07 -0700 Subject: [PATCH 21/81] fix CI --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8fc3bdc7..16057711 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -3,7 +3,7 @@ name: Lint on: push: branches: [ main ] - pull_request: + workflow_call: workflow_dispatch: jobs: From 8bbae1a1a324e48d7567e98395fc2c92840eedc0 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 11 Aug 2025 14:02:29 -0700 Subject: [PATCH 22/81] split out integ tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 80baca1d..47ace280 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ test: test-unit test-integration # Run unit tests test-unit: - uv run pytest test/ --verbose + uv run pytest test/ --ignore=test/integration/ --verbose # Run integration tests test-integration: From 090c8a437af03c579540fdb773d514bdafc17130 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 11 Aug 2025 14:41:37 -0700 Subject: [PATCH 23/81] match boto3, use abc --- src/s3_encryption/__init__.py | 28 ++++++++++++------- .../materials/crypto_materials_manager.py | 9 ++++-- src/s3_encryption/materials/keyring.py | 9 ++++-- test/integration/test_i_s3_encryption.py | 2 +- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index c19ca28d..163ca64e 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -31,23 +31,28 @@ class S3EncryptionClient: wrapped_s3_client = field() config: S3EncryptionClientConfig = field() - # TODO: rename Data-> Body to match boto - def put_object(self, Bucket, Key, Data, EncryptionContext=None, **kwargs): + def put_object(self, **kwargs): + # Extract required parameters from kwargs + bucket = kwargs.pop("Bucket") + key = kwargs.pop("Key") + body = kwargs.pop("Body") + encryption_context = kwargs.pop("EncryptionContext", None) + # Create a pipeline for this operation pipeline = PutEncryptedObjectPipeline(self.config.cmm) # Encrypt the data using the pipeline - data_bytes = Data + data_bytes = body # We probably just shouldn't support strings, use utf8 for now # TODO: look deeper into this, what does normal boto3 do? - if type(Data) == str: - data_bytes = Data.encode("utf-8") + if type(body) == str: + data_bytes = body.encode("utf-8") encrypted_data, encryption_metadata = pipeline.encrypt( - data_bytes, encryption_context=EncryptionContext + data_bytes, encryption_context=encryption_context ) # Add encryption metadata to the request parameters - params = {"Bucket": Bucket, "Key": Key, "Body": encrypted_data, **kwargs} + params = {"Bucket": bucket, "Key": key, "Body": encrypted_data, **kwargs} # Add encryption metadata to the parameters if encryption_metadata: @@ -58,8 +63,11 @@ def put_object(self, Bucket, Key, Data, EncryptionContext=None, **kwargs): return self.wrapped_s3_client.put_object(**params) - def get_object(self, EncryptionContext=None, **kwargs): - # try just straight kwargs + def get_object(self, **kwargs): + # Extract encryption context if provided + encryption_context = kwargs.pop("EncryptionContext", None) + + # Create params for the S3 client params = {**kwargs} # Get the encrypted object from S3 @@ -70,7 +78,7 @@ def get_object(self, EncryptionContext=None, **kwargs): # Decrypt the data using the pipeline decrypted_data = pipeline.decrypt( - response, EncryptionContext + response, encryption_context ) # encrypted_data, encryption_metadata) # Create a new streaming body with the decrypted data diff --git a/src/s3_encryption/materials/crypto_materials_manager.py b/src/s3_encryption/materials/crypto_materials_manager.py index c668a53e..c4247292 100644 --- a/src/s3_encryption/materials/crypto_materials_manager.py +++ b/src/s3_encryption/materials/crypto_materials_manager.py @@ -1,6 +1,7 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +import abc from attrs import define from .keyring import AbstractKeyring @@ -8,7 +9,8 @@ # API Stub for CMM -class AbstractCryptoMaterialsManager: +class AbstractCryptoMaterialsManager(abc.ABC): + @abc.abstractmethod def getEncryptionMaterials(self, encMatsRequest): """Get encryption materials from the keyring. @@ -18,8 +20,9 @@ def getEncryptionMaterials(self, encMatsRequest): Returns: EncryptionMaterials: The encryption materials """ - raise NotImplementedError + pass + @abc.abstractmethod def decryptMaterials(self, decMatsRequest): """Decrypt materials using the keyring. @@ -29,7 +32,7 @@ def decryptMaterials(self, decMatsRequest): Returns: DecryptionMaterials: The decryption materials """ - raise NotImplementedError + pass @define diff --git a/src/s3_encryption/materials/keyring.py b/src/s3_encryption/materials/keyring.py index aed20e96..fe32425b 100644 --- a/src/s3_encryption/materials/keyring.py +++ b/src/s3_encryption/materials/keyring.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 +import abc from attrs import define from ..exceptions import S3EncryptionClientError @@ -9,13 +10,14 @@ @define -class AbstractKeyring: +class AbstractKeyring(abc.ABC): # Ideally, all keyrings would inherit this field. # However, attrs doesn't allow us to set a default here, # when inheriting keyrings have optional fields. # Even without a default it doesn't seem to play nice with attrs. # enableLegacyWrappingAlgorithms: bool = field(default=False) + @abc.abstractmethod def onEncrypt(self, encMaterials): """Process encryption materials. @@ -25,8 +27,9 @@ def onEncrypt(self, encMaterials): Returns: EncryptionMaterials: The processed encryption materials """ - raise NotImplementedError + pass + @abc.abstractmethod def onDecrypt(self, decMaterials, encrypted_data_keys=None): """Decrypt one of the encrypted data keys and update decMaterials. @@ -37,7 +40,7 @@ def onDecrypt(self, decMaterials, encrypted_data_keys=None): Returns: DecryptionMaterials: The updated decMaterials with the plaintext data key (PDK) """ - raise NotImplementedError + pass @define diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index d956f321..f2c4bcf4 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -28,7 +28,7 @@ def test_simple_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) s3ec = S3EncryptionClient(wrapped_client, config) - s3ec.put_object(Bucket=bucket, Key=key, Data=data) + s3ec.put_object(Bucket=bucket, Key=key, Body=data) get_req = {"Bucket": bucket, "Key": key} response = s3ec.get_object(**get_req) output = response["Body"].read().decode("utf-8") From 15d21444c5525f251433d2c6159103115716dc1d Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 11 Aug 2025 15:05:45 -0700 Subject: [PATCH 24/81] snake case renaming --- src/s3_encryption/__init__.py | 2 +- .../materials/crypto_materials_manager.py | 33 +++++++------- src/s3_encryption/materials/keyring.py | 43 ++++++++++--------- src/s3_encryption/materials/kms_keyring.py | 38 ++++++++-------- src/s3_encryption/pipelines.py | 4 +- test/test_decryption_materials_integration.py | 40 ++++++++--------- test/test_encryption_materials_integration.py | 28 ++++++------ 7 files changed, 95 insertions(+), 93 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 163ca64e..0b56454b 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -66,7 +66,7 @@ def put_object(self, **kwargs): def get_object(self, **kwargs): # Extract encryption context if provided encryption_context = kwargs.pop("EncryptionContext", None) - + # Create params for the S3 client params = {**kwargs} diff --git a/src/s3_encryption/materials/crypto_materials_manager.py b/src/s3_encryption/materials/crypto_materials_manager.py index c4247292..06965eff 100644 --- a/src/s3_encryption/materials/crypto_materials_manager.py +++ b/src/s3_encryption/materials/crypto_materials_manager.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import abc + from attrs import define from .keyring import AbstractKeyring @@ -11,11 +12,11 @@ # API Stub for CMM class AbstractCryptoMaterialsManager(abc.ABC): @abc.abstractmethod - def getEncryptionMaterials(self, encMatsRequest): + def get_encryption_materials(self, enc_mats_request): """Get encryption materials from the keyring. Args: - encMatsRequest (Dict[str, Any] or EncryptionMaterials): Request containing encryption parameters + enc_mats_request (Dict[str, Any] or EncryptionMaterials): Request containing encryption parameters Returns: EncryptionMaterials: The encryption materials @@ -23,11 +24,11 @@ def getEncryptionMaterials(self, encMatsRequest): pass @abc.abstractmethod - def decryptMaterials(self, decMatsRequest): + def decrypt_materials(self, dec_mats_request): """Decrypt materials using the keyring. Args: - decMatsRequest (Dict[str, Any] or DecryptionMaterials): Request containing decryption parameters + dec_mats_request (Dict[str, Any] or DecryptionMaterials): Request containing decryption parameters Returns: DecryptionMaterials: The decryption materials @@ -39,39 +40,39 @@ def decryptMaterials(self, decMatsRequest): class DefaultCryptoMaterialsManager(AbstractCryptoMaterialsManager): keyring: AbstractKeyring - def getEncryptionMaterials(self, encMatsRequest): + def get_encryption_materials(self, enc_mats_request): """Get encryption materials from the keyring. Args: - encMatsRequest (Dict[str, Any]): Request containing encryption parameters + enc_mats_request (Dict[str, Any]): Request containing encryption parameters Returns: EncryptionMaterials: The encryption materials """ # Convert dictionary to EncryptionMaterials if needed - if isinstance(encMatsRequest, dict): + if isinstance(enc_mats_request, dict): materials = EncryptionMaterials( - encryption_context=encMatsRequest.get("encryption_context", {}) + encryption_context=enc_mats_request.get("encryption_context", {}) ) else: - materials = encMatsRequest + materials = enc_mats_request - return self.keyring.onEncrypt(materials) + return self.keyring.on_encrypt(materials) - def decryptMaterials(self, decMatsRequest): + def decrypt_materials(self, dec_mats_request): """Decrypt materials using the keyring. Args: - decMatsRequest (Dict[str, Any] or DecryptionMaterials): Request containing decryption parameters + dec_mats_request (Dict[str, Any] or DecryptionMaterials): Request containing decryption parameters Returns: DecryptionMaterials: The decryption materials """ # Convert dictionary to DecryptionMaterials if needed - if isinstance(decMatsRequest, dict): - materials = DecryptionMaterials.from_dict(decMatsRequest) + if isinstance(dec_mats_request, dict): + materials = DecryptionMaterials.from_dict(dec_mats_request) else: - materials = decMatsRequest + materials = dec_mats_request encrypted_data_keys = materials.encrypted_data_keys - return self.keyring.onDecrypt(materials, encrypted_data_keys) + return self.keyring.on_decrypt(materials, encrypted_data_keys) diff --git a/src/s3_encryption/materials/keyring.py b/src/s3_encryption/materials/keyring.py index fe32425b..1f1e5e83 100644 --- a/src/s3_encryption/materials/keyring.py +++ b/src/s3_encryption/materials/keyring.py @@ -3,6 +3,7 @@ import abc + from attrs import define from ..exceptions import S3EncryptionClientError @@ -18,11 +19,11 @@ class AbstractKeyring(abc.ABC): # enableLegacyWrappingAlgorithms: bool = field(default=False) @abc.abstractmethod - def onEncrypt(self, encMaterials): + def on_encrypt(self, enc_materials): """Process encryption materials. Args: - encMaterials (EncryptionMaterials): Encryption materials to process + enc_materials (EncryptionMaterials): Encryption materials to process Returns: EncryptionMaterials: The processed encryption materials @@ -30,15 +31,15 @@ def onEncrypt(self, encMaterials): pass @abc.abstractmethod - def onDecrypt(self, decMaterials, encrypted_data_keys=None): - """Decrypt one of the encrypted data keys and update decMaterials. + def on_decrypt(self, dec_materials, encrypted_data_keys=None): + """Decrypt one of the encrypted data keys and update dec_materials. Args: - decMaterials (DecryptionMaterials): A DecryptionMaterials instance containing decryption materials + dec_materials (DecryptionMaterials): A DecryptionMaterials instance containing decryption materials encrypted_data_keys (List[EncryptedDataKey], optional): A list of encrypted data keys to try. Returns: - DecryptionMaterials: The updated decMaterials with the plaintext data key (PDK) + DecryptionMaterials: The updated dec_materials with the plaintext data key (PDK) """ pass @@ -50,52 +51,52 @@ class S3Keyring(AbstractKeyring): # Ideally this would be set, but attrs doesn't play nice # enable_legacy_wrapping_algorithms: bool = field(default=False) - def onEncrypt(self, encMaterials): + def on_encrypt(self, enc_materials): """Validate encryption materials before encryption. Args: - encMaterials (EncryptionMaterials or dict): Encryption materials + enc_materials (EncryptionMaterials or dict): Encryption materials Returns: EncryptionMaterials: The validated encryption materials """ # Convert dict to EncryptionMaterials if needed - if isinstance(encMaterials, dict): - encMaterials = EncryptionMaterials.from_dict(encMaterials) + if isinstance(enc_materials, dict): + enc_materials = EncryptionMaterials.from_dict(enc_materials) # Validate encryption materials - if not isinstance(encMaterials, EncryptionMaterials): + if not isinstance(enc_materials, EncryptionMaterials): raise S3EncryptionClientError( "Encryption materials must be an EncryptionMaterials instance or a dictionary" ) # Ensure encryption_context is a dictionary - if not isinstance(encMaterials.encryption_context, dict): + if not isinstance(enc_materials.encryption_context, dict): raise S3EncryptionClientError("Encryption context must be a dictionary") - return encMaterials + return enc_materials - def onDecrypt(self, decMaterials, encrypted_data_keys=None): + def on_decrypt(self, dec_materials, encrypted_data_keys=None): """Validate decryption materials before decryption. Args: - decMaterials (DecryptionMaterials): A DecryptionMaterials instance containing decryption materials + dec_materials (DecryptionMaterials): A DecryptionMaterials instance containing decryption materials encrypted_data_keys (List[EncryptedDataKey], optional): A list of encrypted data keys to try. Returns: DecryptionMaterials: The validated decryption materials """ # Validate decryption materials - if not isinstance(decMaterials, DecryptionMaterials): + if not isinstance(dec_materials, DecryptionMaterials): raise S3EncryptionClientError( "Decryption materials must be a DecryptionMaterials instance" ) - # Use encrypted_data_keys from parameters if provided, otherwise use from decMaterials + # Use encrypted_data_keys from parameters if provided, otherwise use from dec_materials edks = ( encrypted_data_keys if encrypted_data_keys is not None - else decMaterials.encrypted_data_keys + else dec_materials.encrypted_data_keys ) # Validate encrypted_data_keys @@ -103,10 +104,10 @@ def onDecrypt(self, decMaterials, encrypted_data_keys=None): raise S3EncryptionClientError("No encrypted data keys provided") # Ensure encryption contexts are dictionaries - if not isinstance(decMaterials.encryption_context_from_request, dict): + if not isinstance(dec_materials.encryption_context_from_request, dict): raise S3EncryptionClientError("Encryption context from request must be a dictionary") - if not isinstance(decMaterials.encryption_context_stored, dict): + if not isinstance(dec_materials.encryption_context_stored, dict): raise S3EncryptionClientError("Stored encryption context must be a dictionary") - return decMaterials + return dec_materials diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index 9559ff29..e7f5804e 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -17,21 +17,21 @@ class KmsKeyring(S3Keyring): kms_key_id: str = field() enable_legacy_wrapping_algorithms: bool = field(default=False) - def onEncrypt(self, encMaterials): + def on_encrypt(self, enc_materials): """Process encryption materials using KMS. Args: - encMaterials (EncryptionMaterials): Encryption materials to process + enc_materials (EncryptionMaterials): Encryption materials to process Returns: EncryptionMaterials: The processed encryption materials with KMS-generated keys """ try: # Call parent class validation - encMaterials = super().onEncrypt(encMaterials) + enc_materials = super().on_encrypt(enc_materials) # Add default encryption context - encryption_context = encMaterials.encryption_context + encryption_context = enc_materials.encryption_context encryption_context["aws:x-amz-cek-alg"] = "AES/GCM/NoPadding" response = self.kms_client.generate_data_key( @@ -43,31 +43,31 @@ def onEncrypt(self, encMaterials): key_provider_info="kms+context", encrypted_data_key=response["CiphertextBlob"], ) - encMaterials.encrypted_data_key = encrypted_data_key - encMaterials.plaintext_data_key = response["Plaintext"] - return encMaterials + enc_materials.encrypted_data_key = encrypted_data_key + enc_materials.plaintext_data_key = response["Plaintext"] + return enc_materials except Exception: raise - def onDecrypt(self, decMaterials, encrypted_data_keys=None): - """Decrypt one of the encrypted data keys and update decMaterials. + def on_decrypt(self, dec_materials, encrypted_data_keys=None): + """Decrypt one of the encrypted data keys and update dec_materials. Args: - decMaterials (DecryptionMaterials): A DecryptionMaterials instance containing decryption materials + dec_materials (DecryptionMaterials): A DecryptionMaterials instance containing decryption materials encrypted_data_keys (List[EncryptedDataKey], optional): A list of encrypted data keys to try. Returns: - DecryptionMaterials: The updated decMaterials with the plaintext data key (PDK) + DecryptionMaterials: The updated dec_materials with the plaintext data key (PDK) """ try: # Call parent class validation - decMaterials = super().onDecrypt(decMaterials, encrypted_data_keys) + dec_materials = super().on_decrypt(dec_materials, encrypted_data_keys) - # Use encrypted_data_keys from parameters if provided, otherwise use from decMaterials + # Use encrypted_data_keys from parameters if provided, otherwise use from dec_materials edks = ( encrypted_data_keys if encrypted_data_keys is not None - else decMaterials.encrypted_data_keys + else dec_materials.encrypted_data_keys ) # Try to decrypt each EDK until one succeeds @@ -78,9 +78,9 @@ def onDecrypt(self, decMaterials, encrypted_data_keys=None): edk_bytes = edk.encrypted_data_key if edk.key_provider_info == "kms+context": encryption_context_from_request = ( - decMaterials.encryption_context_from_request + dec_materials.encryption_context_from_request ) - encryption_context_stored = decMaterials.encryption_context_stored + encryption_context_stored = dec_materials.encryption_context_stored # Default EC MUST NOT be passed in via request if KMS_CONTEXT_DEFAULT_KEY in encryption_context_from_request: @@ -112,10 +112,10 @@ def onDecrypt(self, decMaterials, encrypted_data_keys=None): response = self.kms_client.decrypt( KeyId=self.kms_key_id, CiphertextBlob=edk_bytes, - EncryptionContext=decMaterials.encryption_context_stored, + EncryptionContext=dec_materials.encryption_context_stored, ) - decMaterials.plaintext_data_key = response["Plaintext"] - return decMaterials + dec_materials.plaintext_data_key = response["Plaintext"] + return dec_materials except Exception as e: last_exception = e continue diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 44aa4dfa..6ce08661 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -39,7 +39,7 @@ def encrypt(self, plaintext, encryption_context=None): ) # Get encryption materials from the crypto materials manager - enc_mats = self.cmm.getEncryptionMaterials(enc_mats_request) + enc_mats = self.cmm.get_encryption_materials(enc_mats_request) # Generate initialization vector iv = os.urandom(12) @@ -137,7 +137,7 @@ def decrypt(self, response, encryption_context={}): ) # Get decryption materials from the crypto materials manager - dec_materials = self.cmm.decryptMaterials(dec_materials) + dec_materials = self.cmm.decrypt_materials(dec_materials) aesgcm = AESGCM(dec_materials.plaintext_data_key) diff --git a/test/test_decryption_materials_integration.py b/test/test_decryption_materials_integration.py index 7e697e92..7cdbd0d4 100644 --- a/test/test_decryption_materials_integration.py +++ b/test/test_decryption_materials_integration.py @@ -11,8 +11,8 @@ class TestDecryptionMaterialsIntegration(unittest.TestCase): - def test_keyring_onDecrypt(self): - """Test that S3Keyring.onDecrypt properly handles DecryptionMaterials.""" + def test_keyring_on_decrypt(self): + """Test that S3Keyring.on_decrypt properly handles DecryptionMaterials.""" # Create a keyring keyring = S3Keyring() @@ -32,9 +32,9 @@ def test_keyring_onDecrypt(self): ) # Mock the validation method to return the materials - with patch.object(S3Keyring, "onDecrypt", return_value=materials) as mock_onDecrypt: - # Call onDecrypt - result = keyring.onDecrypt(materials, [edk]) + with patch.object(S3Keyring, "on_decrypt", return_value=materials) as mock_on_decrypt: + # Call on_decrypt + result = keyring.on_decrypt(materials, [edk]) # Verify the result is a DecryptionMaterials instance self.assertIsInstance(result, DecryptionMaterials) @@ -43,8 +43,8 @@ def test_keyring_onDecrypt(self): self.assertEqual(result.encryption_context_stored, {"key1": "value1"}) self.assertEqual(result.encryption_context_from_request, {"key2": "value2"}) - def test_keyring_onDecrypt_default_EC(self): - """Test that S3Keyring.onDecrypt properly handles DecryptionMaterials.""" + def test_keyring_on_decrypt_default_EC(self): + """Test that S3Keyring.on_decrypt properly handles DecryptionMaterials.""" # Create a keyring keyring = S3Keyring() @@ -64,9 +64,9 @@ def test_keyring_onDecrypt_default_EC(self): ) # Mock the validation method to return the materials - with patch.object(S3Keyring, "onDecrypt", return_value=materials) as mock_onDecrypt: - # Call onDecrypt - result = keyring.onDecrypt(materials, [edk]) + with patch.object(S3Keyring, "on_decrypt", return_value=materials) as mock_on_decrypt: + # Call on_decrypt + result = keyring.on_decrypt(materials, [edk]) # Verify the result is a DecryptionMaterials instance self.assertIsInstance(result, DecryptionMaterials) @@ -75,8 +75,8 @@ def test_keyring_onDecrypt_default_EC(self): self.assertEqual(result.encryption_context_stored, {}) self.assertEqual(result.encryption_context_from_request, {}) - def test_cmm_decryptMaterials_with_dict(self): - """Test that DefaultCryptoMaterialsManager.decryptMaterials properly handles dictionary input.""" + def test_cmm_decrypt_materials_with_dict(self): + """Test that DefaultCryptoMaterialsManager.decrypt_materials properly handles dictionary input.""" # Create a mock keyring keyring = MagicMock() edk = EncryptedDataKey( @@ -84,7 +84,7 @@ def test_cmm_decryptMaterials_with_dict(self): key_provider_info="kms+context", encrypted_data_key=b"encrypted-data-key", ) - keyring.onDecrypt.return_value = DecryptionMaterials( + keyring.on_decrypt.return_value = DecryptionMaterials( iv=b"initialization-vector", encrypted_data_keys=[edk], encryption_context_stored={"key1": "value1"}, @@ -95,8 +95,8 @@ def test_cmm_decryptMaterials_with_dict(self): # Create a CMM cmm = DefaultCryptoMaterialsManager(keyring=keyring) - # Call decryptMaterials with a dictionary - result = cmm.decryptMaterials( + # Call decrypt_materials with a dictionary + result = cmm.decrypt_materials( { "iv": b"initialization-vector", "encrypted_data_keys": [edk], @@ -113,8 +113,8 @@ def test_cmm_decryptMaterials_with_dict(self): self.assertEqual(result.encryption_context_from_request, {"key2": "value2"}) self.assertEqual(result.plaintext_data_key, b"plaintext-data-key") - def test_cmm_decryptMaterials_with_materials(self): - """Test that DefaultCryptoMaterialsManager.decryptMaterials properly handles DecryptionMaterials input.""" + def test_cmm_decrypt_materials_with_materials(self): + """Test that DefaultCryptoMaterialsManager.decrypt_materials properly handles DecryptionMaterials input.""" # Create a mock keyring keyring = MagicMock() edk = EncryptedDataKey( @@ -122,7 +122,7 @@ def test_cmm_decryptMaterials_with_materials(self): key_provider_info="kms+context", encrypted_data_key=b"encrypted-data-key", ) - keyring.onDecrypt.return_value = DecryptionMaterials( + keyring.on_decrypt.return_value = DecryptionMaterials( iv=b"initialization-vector", encrypted_data_keys=[edk], encryption_context_stored={"key1": "value1"}, @@ -133,14 +133,14 @@ def test_cmm_decryptMaterials_with_materials(self): # Create a CMM cmm = DefaultCryptoMaterialsManager(keyring=keyring) - # Call decryptMaterials with a DecryptionMaterials instance + # Call decrypt_materials with a DecryptionMaterials instance materials = DecryptionMaterials( iv=b"initialization-vector", encrypted_data_keys=[edk], encryption_context_stored={"key1": "value1"}, encryption_context_from_request={"key2": "value2"}, ) - result = cmm.decryptMaterials(materials) + result = cmm.decrypt_materials(materials) # Verify the result is a DecryptionMaterials instance self.assertIsInstance(result, DecryptionMaterials) diff --git a/test/test_encryption_materials_integration.py b/test/test_encryption_materials_integration.py index be16662d..28356fa8 100644 --- a/test/test_encryption_materials_integration.py +++ b/test/test_encryption_materials_integration.py @@ -11,26 +11,26 @@ class TestEncryptionMaterialsIntegration(unittest.TestCase): - def test_keyring_onEncrypt(self): - """Test that S3Keyring.onEncrypt properly handles EncryptionMaterials.""" + def test_keyring_on_encrypt(self): + """Test that S3Keyring.on_encrypt properly handles EncryptionMaterials.""" # Create a keyring keyring = S3Keyring() # Create encryption materials materials = EncryptionMaterials(encryption_context={"key1": "value1"}) - # Call onEncrypt - result = keyring.onEncrypt(materials) + # Call on_encrypt + result = keyring.on_encrypt(materials) # Verify the result is an EncryptionMaterials instance self.assertIsInstance(result, EncryptionMaterials) self.assertEqual(result.encryption_context, {"key1": "value1"}) - def test_cmm_getEncryptionMaterials_with_dict(self): - """Test that DefaultCryptoMaterialsManager.getEncryptionMaterials properly handles dictionary input.""" + def test_cmm_get_encryption_materials_with_dict(self): + """Test that DefaultCryptoMaterialsManager.get_encryption_materials properly handles dictionary input.""" # Create a mock keyring keyring = MagicMock() - keyring.onEncrypt.return_value = EncryptionMaterials( + keyring.on_encrypt.return_value = EncryptionMaterials( encryption_context={"key1": "value1"}, encrypted_data_key=EncryptedDataKey( key_provider_id=b"S3Keyring", @@ -43,8 +43,8 @@ def test_cmm_getEncryptionMaterials_with_dict(self): # Create a CMM cmm = DefaultCryptoMaterialsManager(keyring=keyring) - # Call getEncryptionMaterials with a dictionary - result = cmm.getEncryptionMaterials({"encryption_context": {"key1": "value1"}}) + # Call get_encryption_materials with a dictionary + result = cmm.get_encryption_materials({"encryption_context": {"key1": "value1"}}) # Verify the result is an EncryptionMaterials instance self.assertIsInstance(result, EncryptionMaterials) @@ -52,11 +52,11 @@ def test_cmm_getEncryptionMaterials_with_dict(self): self.assertIsNotNone(result.encrypted_data_key) self.assertIsNotNone(result.plaintext_data_key) - def test_cmm_getEncryptionMaterials_with_materials(self): - """Test that DefaultCryptoMaterialsManager.getEncryptionMaterials properly handles EncryptionMaterials input.""" + def test_cmm_get_encryption_materials_with_materials(self): + """Test that DefaultCryptoMaterialsManager.get_encryption_materials properly handles EncryptionMaterials input.""" # Create a mock keyring keyring = MagicMock() - keyring.onEncrypt.return_value = EncryptionMaterials( + keyring.on_encrypt.return_value = EncryptionMaterials( encryption_context={"key1": "value1"}, encrypted_data_key=EncryptedDataKey( key_provider_id=b"S3Keyring", @@ -69,9 +69,9 @@ def test_cmm_getEncryptionMaterials_with_materials(self): # Create a CMM cmm = DefaultCryptoMaterialsManager(keyring=keyring) - # Call getEncryptionMaterials with an EncryptionMaterials instance + # Call get_encryption_materials with an EncryptionMaterials instance materials = EncryptionMaterials(encryption_context={"key1": "value1"}) - result = cmm.getEncryptionMaterials(materials) + result = cmm.get_encryption_materials(materials) # Verify the result is an EncryptionMaterials instance self.assertIsInstance(result, EncryptionMaterials) From cac76ce1a15988ef09ed4900ce74cd6f7729b3b5 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 11 Aug 2025 15:07:18 -0700 Subject: [PATCH 25/81] format --- src/s3_encryption/materials/kms_keyring.py | 3 +-- test/integration/test_i_s3_encryption.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index e7f5804e..e5da0593 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -123,7 +123,6 @@ def on_decrypt(self, dec_materials, encrypted_data_keys=None): # If we get here, none of the EDKs could be decrypted if last_exception: raise last_exception - else: - raise S3EncryptionClientError("Failed to decrypt any of the encrypted data keys") + raise S3EncryptionClientError("Failed to decrypt any of the encrypted data keys") except Exception: raise diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index f2c4bcf4..14692c41 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -39,5 +39,4 @@ def test_simple_roundtrip(): print("Output:") print(output) raise RuntimeError - else: - print("Success!") + print("Success!") From f231ab2e6b3eb8569a14b3dfa5798eca3ad2e9ec Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 11 Aug 2025 15:10:07 -0700 Subject: [PATCH 26/81] remove lock file, format, simplify attrs --- .gitignore | 4 + poetry.lock | 656 ------------------ .../materials/encrypted_data_key.py | 8 +- src/s3_encryption/materials/kms_keyring.py | 3 +- test/test_decryption_materials_integration.py | 2 +- 5 files changed, 11 insertions(+), 662 deletions(-) delete mode 100644 poetry.lock diff --git a/.gitignore b/.gitignore index 4b1c0c91..22b9a5f7 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ __pycache__/ dist/ build/ *.egg-info/ + +# Uv +.uv/ +uv.lock diff --git a/poetry.lock b/poetry.lock deleted file mode 100644 index a89805ee..00000000 --- a/poetry.lock +++ /dev/null @@ -1,656 +0,0 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. - -[[package]] -name = "attrs" -version = "25.3.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=3.8" -files = [ - {file = "attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3"}, - {file = "attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b"}, -] - -[package.extras] -benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier"] -tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] - -[[package]] -name = "aws-cryptographic-material-providers" -version = "1.11.0" -description = "AWS Cryptographic Material Providers Library for Python" -optional = false -python-versions = "<4.0.0,>=3.11.0" -files = [ - {file = "aws_cryptographic_material_providers-1.11.0-py3-none-any.whl", hash = "sha256:9a9f0dca5b1902a4f16fb91cc1010dee74a721f84f411e81ffb4481fc0dd095f"}, - {file = "aws_cryptographic_material_providers-1.11.0.tar.gz", hash = "sha256:4ea5f9e5cc003e97d2ef98079dc25d8c49a0db01315ee887d19fd2f1c85ae9c3"}, -] - -[package.dependencies] -aws-cryptography-internal-dynamodb = "1.11.0" -aws-cryptography-internal-kms = "1.11.0" -aws-cryptography-internal-primitives = "1.11.0" -aws-cryptography-internal-standard-library = "1.11.0" - -[[package]] -name = "aws-cryptography-internal-dynamodb" -version = "1.11.0" -description = "" -optional = false -python-versions = "<4.0.0,>=3.11.0" -files = [ - {file = "aws_cryptography_internal_dynamodb-1.11.0-py3-none-any.whl", hash = "sha256:5a2da0ae6829d725f24018d001f4c733605f213820b723b6c75015843dc2427c"}, - {file = "aws_cryptography_internal_dynamodb-1.11.0.tar.gz", hash = "sha256:0800921ebb5dafc2853a2f5449f74aa03d24acd9ddb2ee58edca4002b97a5da5"}, -] - -[package.dependencies] -aws-cryptography-internal-standard-library = "1.11.0" -boto3 = ">=1.35.42,<2.0.0" - -[[package]] -name = "aws-cryptography-internal-kms" -version = "1.11.0" -description = "" -optional = false -python-versions = "<4.0.0,>=3.11.0" -files = [ - {file = "aws_cryptography_internal_kms-1.11.0-py3-none-any.whl", hash = "sha256:1c23cc8e970252fc7627868fc6b7a002400ec1d555ac29368e0eaddcceb07953"}, - {file = "aws_cryptography_internal_kms-1.11.0.tar.gz", hash = "sha256:a3ff5105b3e1c9d81e9698e0efc80de8a6bb8078b4512f9b39ed0f6161aae172"}, -] - -[package.dependencies] -aws-cryptography-internal-standard-library = "1.11.0" -boto3 = ">=1.35.42,<2.0.0" - -[[package]] -name = "aws-cryptography-internal-primitives" -version = "1.11.0" -description = "" -optional = false -python-versions = "<4.0.0,>=3.11.0" -files = [ - {file = "aws_cryptography_internal_primitives-1.11.0-py3-none-any.whl", hash = "sha256:84200885113f3534f4bff819ac1603c6d5c3bdd4d5c83a1b73ac2462cecec49b"}, - {file = "aws_cryptography_internal_primitives-1.11.0.tar.gz", hash = "sha256:9072af2c403b9e729dc767b44d1d642fa924a317a5bdbdffdf6dba0e93dc7996"}, -] - -[package.dependencies] -aws-cryptography-internal-standard-library = "1.11.0" -cryptography = ">=43.0.1,<46" - -[[package]] -name = "aws-cryptography-internal-standard-library" -version = "1.11.0" -description = "" -optional = false -python-versions = "<4.0.0,>=3.11.0" -files = [ - {file = "aws_cryptography_internal_standard_library-1.11.0-py3-none-any.whl", hash = "sha256:a2d5a4d8f70bce7242e8ebe06742223b8cd93253ed8081f44d7a8c1a086871e1"}, - {file = "aws_cryptography_internal_standard_library-1.11.0.tar.gz", hash = "sha256:36d82c6bc0361cf0ec3b7181804d375718f5c297949ddd902670f4452ecad3b0"}, -] - -[package.dependencies] -DafnyRuntimePython = "4.9.0" -pytz = ">=2023.3.post1,<2025.0.0" - -[[package]] -name = "black" -version = "24.10.0" -description = "The uncompromising code formatter." -optional = false -python-versions = ">=3.9" -files = [ - {file = "black-24.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6668650ea4b685440857138e5fe40cde4d652633b1bdffc62933d0db4ed9812"}, - {file = "black-24.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1c536fcf674217e87b8cc3657b81809d3c085d7bf3ef262ead700da345bfa6ea"}, - {file = "black-24.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:649fff99a20bd06c6f727d2a27f401331dc0cc861fb69cde910fe95b01b5928f"}, - {file = "black-24.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:fe4d6476887de70546212c99ac9bd803d90b42fc4767f058a0baa895013fbb3e"}, - {file = "black-24.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5a2221696a8224e335c28816a9d331a6c2ae15a2ee34ec857dcf3e45dbfa99ad"}, - {file = "black-24.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9da3333530dbcecc1be13e69c250ed8dfa67f43c4005fb537bb426e19200d50"}, - {file = "black-24.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4007b1393d902b48b36958a216c20c4482f601569d19ed1df294a496eb366392"}, - {file = "black-24.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:394d4ddc64782e51153eadcaaca95144ac4c35e27ef9b0a42e121ae7e57a9175"}, - {file = "black-24.10.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b5e39e0fae001df40f95bd8cc36b9165c5e2ea88900167bddf258bacef9bbdc3"}, - {file = "black-24.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d37d422772111794b26757c5b55a3eade028aa3fde43121ab7b673d050949d65"}, - {file = "black-24.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14b3502784f09ce2443830e3133dacf2c0110d45191ed470ecb04d0f5f6fcb0f"}, - {file = "black-24.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:30d2c30dc5139211dda799758559d1b049f7f14c580c409d6ad925b74a4208a8"}, - {file = "black-24.10.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cbacacb19e922a1d75ef2b6ccaefcd6e93a2c05ede32f06a21386a04cedb981"}, - {file = "black-24.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1f93102e0c5bb3907451063e08b9876dbeac810e7da5a8bfb7aeb5a9ef89066b"}, - {file = "black-24.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ddacb691cdcdf77b96f549cf9591701d8db36b2f19519373d60d31746068dbf2"}, - {file = "black-24.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:680359d932801c76d2e9c9068d05c6b107f2584b2a5b88831c83962eb9984c1b"}, - {file = "black-24.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:17374989640fbca88b6a448129cd1745c5eb8d9547b464f281b251dd00155ccd"}, - {file = "black-24.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:63f626344343083322233f175aaf372d326de8436f5928c042639a4afbbf1d3f"}, - {file = "black-24.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfa1d0cb6200857f1923b602f978386a3a2758a65b52e0950299ea014be6800"}, - {file = "black-24.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:2cd9c95431d94adc56600710f8813ee27eea544dd118d45896bb734e9d7a0dc7"}, - {file = "black-24.10.0-py3-none-any.whl", hash = "sha256:3bb2b7a1f7b685f85b11fed1ef10f8a9148bceb49853e47a294a3dd963c1dd7d"}, - {file = "black-24.10.0.tar.gz", hash = "sha256:846ea64c97afe3bc677b761787993be4991810ecc7a4a937816dd6bddedc4875"}, -] - -[package.dependencies] -click = ">=8.0.0" -mypy-extensions = ">=0.4.3" -packaging = ">=22.0" -pathspec = ">=0.9.0" -platformdirs = ">=2" - -[package.extras] -colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.10)"] -jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] -uvloop = ["uvloop (>=0.15.2)"] - -[[package]] -name = "boto3" -version = "1.39.14" -description = "The AWS SDK for Python" -optional = false -python-versions = ">=3.9" -files = [ - {file = "boto3-1.39.14-py3-none-any.whl", hash = "sha256:82c6868cad18c3bd4170915e9525f9af5f83e9779c528417f8863629558fc2d0"}, - {file = "boto3-1.39.14.tar.gz", hash = "sha256:fabb16360a93b449d5241006485bcc761c26694e75ac01009f4459f114acc06e"}, -] - -[package.dependencies] -botocore = ">=1.39.14,<1.40.0" -jmespath = ">=0.7.1,<2.0.0" -s3transfer = ">=0.13.0,<0.14.0" - -[package.extras] -crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] - -[[package]] -name = "botocore" -version = "1.39.14" -description = "Low-level, data-driven core of boto 3." -optional = false -python-versions = ">=3.9" -files = [ - {file = "botocore-1.39.14-py3-none-any.whl", hash = "sha256:4ed551c77194167b7e8063f33059bc2f9b2ead0ed4ee33dc7857273648ed4349"}, - {file = "botocore-1.39.14.tar.gz", hash = "sha256:7fc44d4ad13b524e5d8a6296785776ef5898ac026ff74df9b35313831d507926"}, -] - -[package.dependencies] -jmespath = ">=0.7.1,<2.0.0" -python-dateutil = ">=2.1,<3.0.0" -urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} - -[package.extras] -crt = ["awscrt (==0.23.8)"] - -[[package]] -name = "cffi" -version = "1.17.1" -description = "Foreign Function Interface for Python calling C code." -optional = false -python-versions = ">=3.8" -files = [ - {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, - {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, - {file = "cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382"}, - {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702"}, - {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3"}, - {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6"}, - {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17"}, - {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8"}, - {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e"}, - {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be"}, - {file = "cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c"}, - {file = "cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15"}, - {file = "cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401"}, - {file = "cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf"}, - {file = "cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4"}, - {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41"}, - {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1"}, - {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6"}, - {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d"}, - {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6"}, - {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f"}, - {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"}, - {file = "cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655"}, - {file = "cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0"}, - {file = "cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4"}, - {file = "cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c"}, - {file = "cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36"}, - {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5"}, - {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff"}, - {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99"}, - {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93"}, - {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3"}, - {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8"}, - {file = "cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65"}, - {file = "cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903"}, - {file = "cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e"}, - {file = "cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2"}, - {file = "cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3"}, - {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683"}, - {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5"}, - {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4"}, - {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd"}, - {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed"}, - {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9"}, - {file = "cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d"}, - {file = "cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a"}, - {file = "cffi-1.17.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b"}, - {file = "cffi-1.17.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964"}, - {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9"}, - {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc"}, - {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c"}, - {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1"}, - {file = "cffi-1.17.1-cp38-cp38-win32.whl", hash = "sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8"}, - {file = "cffi-1.17.1-cp38-cp38-win_amd64.whl", hash = "sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1"}, - {file = "cffi-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16"}, - {file = "cffi-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36"}, - {file = "cffi-1.17.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8"}, - {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576"}, - {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87"}, - {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0"}, - {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3"}, - {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595"}, - {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a"}, - {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e"}, - {file = "cffi-1.17.1-cp39-cp39-win32.whl", hash = "sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7"}, - {file = "cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662"}, - {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"}, -] - -[package.dependencies] -pycparser = "*" - -[[package]] -name = "click" -version = "8.2.1" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.10" -files = [ - {file = "click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b"}, - {file = "click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "cryptography" -version = "43.0.3" -description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -optional = false -python-versions = ">=3.7" -files = [ - {file = "cryptography-43.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e"}, - {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e"}, - {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e1ce50266f4f70bf41a2c6dc4358afadae90e2a1e5342d3c08883df1675374f"}, - {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:443c4a81bb10daed9a8f334365fe52542771f25aedaf889fd323a853ce7377d6"}, - {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:74f57f24754fe349223792466a709f8e0c093205ff0dca557af51072ff47ab18"}, - {file = "cryptography-43.0.3-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9762ea51a8fc2a88b70cf2995e5675b38d93bf36bd67d91721c309df184f49bd"}, - {file = "cryptography-43.0.3-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:81ef806b1fef6b06dcebad789f988d3b37ccaee225695cf3e07648eee0fc6b73"}, - {file = "cryptography-43.0.3-cp37-abi3-win32.whl", hash = "sha256:cbeb489927bd7af4aa98d4b261af9a5bc025bd87f0e3547e11584be9e9427be2"}, - {file = "cryptography-43.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:f46304d6f0c6ab8e52770addfa2fc41e6629495548862279641972b6215451cd"}, - {file = "cryptography-43.0.3-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:8ac43ae87929a5982f5948ceda07001ee5e83227fd69cf55b109144938d96984"}, - {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:846da004a5804145a5f441b8530b4bf35afbf7da70f82409f151695b127213d5"}, - {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f996e7268af62598f2fc1204afa98a3b5712313a55c4c9d434aef49cadc91d4"}, - {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f7b178f11ed3664fd0e995a47ed2b5ff0a12d893e41dd0494f406d1cf555cab7"}, - {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:c2e6fc39c4ab499049df3bdf567f768a723a5e8464816e8f009f121a5a9f4405"}, - {file = "cryptography-43.0.3-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e1be4655c7ef6e1bbe6b5d0403526601323420bcf414598955968c9ef3eb7d16"}, - {file = "cryptography-43.0.3-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:df6b6c6d742395dd77a23ea3728ab62f98379eff8fb61be2744d4679ab678f73"}, - {file = "cryptography-43.0.3-cp39-abi3-win32.whl", hash = "sha256:d56e96520b1020449bbace2b78b603442e7e378a9b3bd68de65c782db1507995"}, - {file = "cryptography-43.0.3-cp39-abi3-win_amd64.whl", hash = "sha256:0c580952eef9bf68c4747774cde7ec1d85a6e61de97281f2dba83c7d2c806362"}, - {file = "cryptography-43.0.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:d03b5621a135bffecad2c73e9f4deb1a0f977b9a8ffe6f8e002bf6c9d07b918c"}, - {file = "cryptography-43.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a2a431ee15799d6db9fe80c82b055bae5a752bef645bba795e8e52687c69efe3"}, - {file = "cryptography-43.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:281c945d0e28c92ca5e5930664c1cefd85efe80e5c0d2bc58dd63383fda29f83"}, - {file = "cryptography-43.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f18c716be16bc1fea8e95def49edf46b82fccaa88587a45f8dc0ff6ab5d8e0a7"}, - {file = "cryptography-43.0.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:4a02ded6cd4f0a5562a8887df8b3bd14e822a90f97ac5e544c162899bc467664"}, - {file = "cryptography-43.0.3-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:53a583b6637ab4c4e3591a15bc9db855b8d9dee9a669b550f311480acab6eb08"}, - {file = "cryptography-43.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1ec0bcf7e17c0c5669d881b1cd38c4972fade441b27bda1051665faaa89bdcaa"}, - {file = "cryptography-43.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2ce6fae5bdad59577b44e4dfed356944fbf1d925269114c28be377692643b4ff"}, - {file = "cryptography-43.0.3.tar.gz", hash = "sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805"}, -] - -[package.dependencies] -cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} - -[package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] -docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] -nox = ["nox"] -pep8test = ["check-sdist", "click", "mypy", "ruff"] -sdist = ["build"] -ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi", "cryptography-vectors (==43.0.3)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] -test-randomorder = ["pytest-randomly"] - -[[package]] -name = "dafnyruntimepython" -version = "4.9.0" -description = "Dafny runtime for Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "DafnyRuntimePython-4.9.0-py3-none-any.whl", hash = "sha256:c9cdcf127f5b6a4c6c9cf69016b9486318c3a6600e7f03fcbc621f6a5398479c"}, - {file = "dafnyruntimepython-4.9.0.tar.gz", hash = "sha256:03a4c2dbbe45c13dc2c7dbefad01812367b3bb217a14b4b848d7e94ef5c08cee"}, -] - -[[package]] -name = "flake8" -version = "7.3.0" -description = "the modular source code checker: pep8 pyflakes and co" -optional = false -python-versions = ">=3.9" -files = [ - {file = "flake8-7.3.0-py2.py3-none-any.whl", hash = "sha256:b9696257b9ce8beb888cdbe31cf885c90d31928fe202be0889a7cdafad32f01e"}, - {file = "flake8-7.3.0.tar.gz", hash = "sha256:fe044858146b9fc69b551a4b490d69cf960fcb78ad1edcb84e7fbb1b4a8e3872"}, -] - -[package.dependencies] -mccabe = ">=0.7.0,<0.8.0" -pycodestyle = ">=2.14.0,<2.15.0" -pyflakes = ">=3.4.0,<3.5.0" - -[[package]] -name = "flake8-docstrings" -version = "1.7.0" -description = "Extension for flake8 which uses pydocstyle to check docstrings" -optional = false -python-versions = ">=3.7" -files = [ - {file = "flake8_docstrings-1.7.0-py2.py3-none-any.whl", hash = "sha256:51f2344026da083fc084166a9353f5082b01f72901df422f74b4d953ae88ac75"}, - {file = "flake8_docstrings-1.7.0.tar.gz", hash = "sha256:4c8cc748dc16e6869728699e5d0d685da9a10b0ea718e090b1ba088e67a941af"}, -] - -[package.dependencies] -flake8 = ">=3" -pydocstyle = ">=2.1" - -[[package]] -name = "iniconfig" -version = "2.1.0" -description = "brain-dead simple config-ini parsing" -optional = false -python-versions = ">=3.8" -files = [ - {file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"}, - {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, -] - -[[package]] -name = "isort" -version = "5.13.2" -description = "A Python utility / library to sort Python imports." -optional = false -python-versions = ">=3.8.0" -files = [ - {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, - {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, -] - -[package.extras] -colors = ["colorama (>=0.4.6)"] - -[[package]] -name = "jmespath" -version = "1.0.1" -description = "JSON Matching Expressions" -optional = false -python-versions = ">=3.7" -files = [ - {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, - {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, -] - -[[package]] -name = "mccabe" -version = "0.7.0" -description = "McCabe checker, plugin for flake8" -optional = false -python-versions = ">=3.6" -files = [ - {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, - {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, -] - -[[package]] -name = "mypy-extensions" -version = "1.1.0" -description = "Type system extensions for programs checked with the mypy type checker." -optional = false -python-versions = ">=3.8" -files = [ - {file = "mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505"}, - {file = "mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558"}, -] - -[[package]] -name = "packaging" -version = "25.0" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -files = [ - {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, - {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, -] - -[[package]] -name = "pathspec" -version = "0.12.1" -description = "Utility library for gitignore style pattern matching of file paths." -optional = false -python-versions = ">=3.8" -files = [ - {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, - {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, -] - -[[package]] -name = "platformdirs" -version = "4.3.8" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." -optional = false -python-versions = ">=3.9" -files = [ - {file = "platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4"}, - {file = "platformdirs-4.3.8.tar.gz", hash = "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc"}, -] - -[package.extras] -docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.1.3)", "sphinx-autodoc-typehints (>=3)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.4)", "pytest-cov (>=6)", "pytest-mock (>=3.14)"] -type = ["mypy (>=1.14.1)"] - -[[package]] -name = "pluggy" -version = "1.6.0" -description = "plugin and hook calling mechanisms for python" -optional = false -python-versions = ">=3.9" -files = [ - {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, - {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, -] - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["coverage", "pytest", "pytest-benchmark"] - -[[package]] -name = "pycodestyle" -version = "2.14.0" -description = "Python style guide checker" -optional = false -python-versions = ">=3.9" -files = [ - {file = "pycodestyle-2.14.0-py2.py3-none-any.whl", hash = "sha256:dd6bf7cb4ee77f8e016f9c8e74a35ddd9f67e1d5fd4184d86c3b98e07099f42d"}, - {file = "pycodestyle-2.14.0.tar.gz", hash = "sha256:c4b5b517d278089ff9d0abdec919cd97262a3367449ea1c8b49b91529167b783"}, -] - -[[package]] -name = "pycparser" -version = "2.22" -description = "C parser in Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, - {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, -] - -[[package]] -name = "pydocstyle" -version = "6.3.0" -description = "Python docstring style checker" -optional = false -python-versions = ">=3.6" -files = [ - {file = "pydocstyle-6.3.0-py3-none-any.whl", hash = "sha256:118762d452a49d6b05e194ef344a55822987a462831ade91ec5c06fd2169d019"}, - {file = "pydocstyle-6.3.0.tar.gz", hash = "sha256:7ce43f0c0ac87b07494eb9c0b462c0b73e6ff276807f204d6b53edc72b7e44e1"}, -] - -[package.dependencies] -snowballstemmer = ">=2.2.0" - -[package.extras] -toml = ["tomli (>=1.2.3)"] - -[[package]] -name = "pyflakes" -version = "3.4.0" -description = "passive checker of Python programs" -optional = false -python-versions = ">=3.9" -files = [ - {file = "pyflakes-3.4.0-py2.py3-none-any.whl", hash = "sha256:f742a7dbd0d9cb9ea41e9a24a918996e8170c799fa528688d40dd582c8265f4f"}, - {file = "pyflakes-3.4.0.tar.gz", hash = "sha256:b24f96fafb7d2ab0ec5075b7350b3d2d2218eab42003821c06344973d3ea2f58"}, -] - -[[package]] -name = "pygments" -version = "2.19.2" -description = "Pygments is a syntax highlighting package written in Python." -optional = false -python-versions = ">=3.8" -files = [ - {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, - {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, -] - -[package.extras] -windows-terminal = ["colorama (>=0.4.6)"] - -[[package]] -name = "pytest" -version = "8.4.1" -description = "pytest: simple powerful testing with Python" -optional = false -python-versions = ">=3.9" -files = [ - {file = "pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7"}, - {file = "pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c"}, -] - -[package.dependencies] -colorama = {version = ">=0.4", markers = "sys_platform == \"win32\""} -iniconfig = ">=1" -packaging = ">=20" -pluggy = ">=1.5,<2" -pygments = ">=2.7.2" - -[package.extras] -dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests", "setuptools", "xmlschema"] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -files = [ - {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, - {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pytz" -version = "2024.2" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -files = [ - {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, - {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, -] - -[[package]] -name = "s3transfer" -version = "0.13.1" -description = "An Amazon S3 Transfer Manager" -optional = false -python-versions = ">=3.9" -files = [ - {file = "s3transfer-0.13.1-py3-none-any.whl", hash = "sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724"}, - {file = "s3transfer-0.13.1.tar.gz", hash = "sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf"}, -] - -[package.dependencies] -botocore = ">=1.37.4,<2.0a.0" - -[package.extras] -crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] - -[[package]] -name = "six" -version = "1.17.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -files = [ - {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, - {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, -] - -[[package]] -name = "snowballstemmer" -version = "3.0.1" -description = "This package provides 32 stemmers for 30 languages generated from Snowball algorithms." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*" -files = [ - {file = "snowballstemmer-3.0.1-py3-none-any.whl", hash = "sha256:6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064"}, - {file = "snowballstemmer-3.0.1.tar.gz", hash = "sha256:6d5eeeec8e9f84d4d56b847692bacf79bc2c8e90c7f80ca4444ff8b6f2e52895"}, -] - -[[package]] -name = "urllib3" -version = "2.5.0" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.9" -files = [ - {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, - {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, -] - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] - -[metadata] -lock-version = "2.0" -python-versions = "^3.11" -content-hash = "e0d80bd0119ad8c72dfd80afa69019310ce4c75a9f81e6da9bb80666657840e2" diff --git a/src/s3_encryption/materials/encrypted_data_key.py b/src/s3_encryption/materials/encrypted_data_key.py index 51d47c3b..057fafda 100644 --- a/src/s3_encryption/materials/encrypted_data_key.py +++ b/src/s3_encryption/materials/encrypted_data_key.py @@ -1,6 +1,6 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from attrs import define, field +from attrs import define @define @@ -16,6 +16,6 @@ class EncryptedDataKey: encrypted_data_key (bytes): The encrypted data key """ - key_provider_info: str = field() - key_provider_id: bytes = field() - encrypted_data_key: bytes = field() + key_provider_info: str + key_provider_id: bytes + encrypted_data_key: bytes diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index e5da0593..27633743 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from attrs import define, field +from botocore import client from ..exceptions import S3EncryptionClientError from .encrypted_data_key import EncryptedDataKey @@ -13,7 +14,7 @@ @define class KmsKeyring(S3Keyring): - kms_client = field() + kms_client = client.BaseClient kms_key_id: str = field() enable_legacy_wrapping_algorithms: bool = field(default=False) diff --git a/test/test_decryption_materials_integration.py b/test/test_decryption_materials_integration.py index 7cdbd0d4..23e65c22 100644 --- a/test/test_decryption_materials_integration.py +++ b/test/test_decryption_materials_integration.py @@ -43,7 +43,7 @@ def test_keyring_on_decrypt(self): self.assertEqual(result.encryption_context_stored, {"key1": "value1"}) self.assertEqual(result.encryption_context_from_request, {"key2": "value2"}) - def test_keyring_on_decrypt_default_EC(self): + def test_keyring_on_decrypt_default_enc_ctx(self): """Test that S3Keyring.on_decrypt properly handles DecryptionMaterials.""" # Create a keyring keyring = S3Keyring() From 2d686bc77c4a57ad76af5f93624a5cdf98c9a3bc Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 11 Aug 2025 15:12:24 -0700 Subject: [PATCH 27/81] remove typehint --- src/s3_encryption/materials/kms_keyring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index 27633743..8b6914da 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -14,7 +14,7 @@ @define class KmsKeyring(S3Keyring): - kms_client = client.BaseClient + kms_client = field() kms_key_id: str = field() enable_legacy_wrapping_algorithms: bool = field(default=False) From aed461ecd07d796507367c947c32d67b3e88bd03 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 11 Aug 2025 15:21:23 -0700 Subject: [PATCH 28/81] use pytest --- test/test_decryption_materials.py | 48 ++++++++--------- test/test_decryption_materials_integration.py | 52 +++++++++---------- test/test_encryption_materials.py | 28 +++++----- test/test_encryption_materials_integration.py | 28 +++++----- test/test_metadata.py | 42 +++++++-------- 5 files changed, 89 insertions(+), 109 deletions(-) diff --git a/test/test_decryption_materials.py b/test/test_decryption_materials.py index 2ada8af8..94cd0c4e 100644 --- a/test/test_decryption_materials.py +++ b/test/test_decryption_materials.py @@ -1,21 +1,21 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -import unittest +import pytest from src.s3_encryption.materials.encrypted_data_key import EncryptedDataKey from src.s3_encryption.materials.materials import DecryptionMaterials -class TestDecryptionMaterials(unittest.TestCase): +class TestDecryptionMaterials: def test_create_decryption_materials(self): """Test creating a DecryptionMaterials instance.""" materials = DecryptionMaterials() - self.assertEqual(materials.encrypted_data_keys, []) - self.assertEqual(materials.encryption_context_stored, {}) - self.assertEqual(materials.encryption_context_from_request, {}) - self.assertIsNone(materials.iv) - self.assertIsNone(materials.plaintext_data_key) + assert materials.encrypted_data_keys == [] + assert materials.encryption_context_stored == {} + assert materials.encryption_context_from_request == {} + assert materials.iv is None + assert materials.plaintext_data_key is None def test_create_with_parameters(self): """Test creating a DecryptionMaterials instance with parameters.""" @@ -39,11 +39,11 @@ def test_create_with_parameters(self): plaintext_data_key=plaintext_data_key, ) - self.assertEqual(materials.iv, iv) - self.assertEqual(materials.encrypted_data_keys, encrypted_data_keys) - self.assertEqual(materials.encryption_context_stored, encryption_context_stored) - self.assertEqual(materials.encryption_context_from_request, encryption_context_from_request) - self.assertEqual(materials.plaintext_data_key, plaintext_data_key) + assert materials.iv == iv + assert materials.encrypted_data_keys == encrypted_data_keys + assert materials.encryption_context_stored == encryption_context_stored + assert materials.encryption_context_from_request == encryption_context_from_request + assert materials.plaintext_data_key == plaintext_data_key def test_from_dict(self): """Test creating a DecryptionMaterials instance from a dictionary.""" @@ -60,11 +60,11 @@ def test_from_dict(self): "PDK": b"plaintext-data-key", } materials = DecryptionMaterials.from_dict(materials_dict) - self.assertEqual(materials.iv, b"initialization-vector") - self.assertEqual(materials.encrypted_data_keys, [edk]) - self.assertEqual(materials.encryption_context_stored, {"key1": "value1"}) - self.assertEqual(materials.encryption_context_from_request, {"key2": "value2"}) - self.assertEqual(materials.plaintext_data_key, b"plaintext-data-key") + assert materials.iv == b"initialization-vector" + assert materials.encrypted_data_keys == [edk] + assert materials.encryption_context_stored == {"key1": "value1"} + assert materials.encryption_context_from_request == {"key2": "value2"} + assert materials.plaintext_data_key == b"plaintext-data-key" def test_to_dict(self): """Test converting a DecryptionMaterials instance to a dictionary.""" @@ -81,12 +81,8 @@ def test_to_dict(self): plaintext_data_key=b"plaintext-data-key", ) materials_dict = materials.to_dict() - self.assertEqual(materials_dict["iv"], b"initialization-vector") - self.assertEqual(materials_dict["encrypted_data_keys"], [edk]) - self.assertEqual(materials_dict["encryption_context_stored"], {"key1": "value1"}) - self.assertEqual(materials_dict["encryption_context_from_request"], {"key2": "value2"}) - self.assertEqual(materials_dict["PDK"], b"plaintext-data-key") - - -if __name__ == "__main__": - unittest.main() + assert materials_dict["iv"] == b"initialization-vector" + assert materials_dict["encrypted_data_keys"] == [edk] + assert materials_dict["encryption_context_stored"] == {"key1": "value1"} + assert materials_dict["encryption_context_from_request"] == {"key2": "value2"} + assert materials_dict["PDK"] == b"plaintext-data-key" diff --git a/test/test_decryption_materials_integration.py b/test/test_decryption_materials_integration.py index 23e65c22..7a20e163 100644 --- a/test/test_decryption_materials_integration.py +++ b/test/test_decryption_materials_integration.py @@ -1,7 +1,7 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -import unittest +import pytest from unittest.mock import MagicMock, patch from src.s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager @@ -10,7 +10,7 @@ from src.s3_encryption.materials.materials import DecryptionMaterials -class TestDecryptionMaterialsIntegration(unittest.TestCase): +class TestDecryptionMaterialsIntegration: def test_keyring_on_decrypt(self): """Test that S3Keyring.on_decrypt properly handles DecryptionMaterials.""" # Create a keyring @@ -37,11 +37,11 @@ def test_keyring_on_decrypt(self): result = keyring.on_decrypt(materials, [edk]) # Verify the result is a DecryptionMaterials instance - self.assertIsInstance(result, DecryptionMaterials) - self.assertEqual(result.iv, b"initialization-vector") - self.assertEqual(result.encrypted_data_keys, [edk]) - self.assertEqual(result.encryption_context_stored, {"key1": "value1"}) - self.assertEqual(result.encryption_context_from_request, {"key2": "value2"}) + assert isinstance(result, DecryptionMaterials) + assert result.iv == b"initialization-vector" + assert result.encrypted_data_keys == [edk] + assert result.encryption_context_stored == {"key1": "value1"} + assert result.encryption_context_from_request == {"key2": "value2"} def test_keyring_on_decrypt_default_enc_ctx(self): """Test that S3Keyring.on_decrypt properly handles DecryptionMaterials.""" @@ -69,11 +69,11 @@ def test_keyring_on_decrypt_default_enc_ctx(self): result = keyring.on_decrypt(materials, [edk]) # Verify the result is a DecryptionMaterials instance - self.assertIsInstance(result, DecryptionMaterials) - self.assertEqual(result.iv, b"initialization-vector") - self.assertEqual(result.encrypted_data_keys, [edk]) - self.assertEqual(result.encryption_context_stored, {}) - self.assertEqual(result.encryption_context_from_request, {}) + assert isinstance(result, DecryptionMaterials) + assert result.iv == b"initialization-vector" + assert result.encrypted_data_keys == [edk] + assert result.encryption_context_stored == {} + assert result.encryption_context_from_request == {} def test_cmm_decrypt_materials_with_dict(self): """Test that DefaultCryptoMaterialsManager.decrypt_materials properly handles dictionary input.""" @@ -106,12 +106,12 @@ def test_cmm_decrypt_materials_with_dict(self): ) # Verify the result is a DecryptionMaterials instance - self.assertIsInstance(result, DecryptionMaterials) - self.assertEqual(result.iv, b"initialization-vector") - self.assertEqual(result.encrypted_data_keys, [edk]) - self.assertEqual(result.encryption_context_stored, {"key1": "value1"}) - self.assertEqual(result.encryption_context_from_request, {"key2": "value2"}) - self.assertEqual(result.plaintext_data_key, b"plaintext-data-key") + assert isinstance(result, DecryptionMaterials) + assert result.iv == b"initialization-vector" + assert result.encrypted_data_keys == [edk] + assert result.encryption_context_stored == {"key1": "value1"} + assert result.encryption_context_from_request == {"key2": "value2"} + assert result.plaintext_data_key == b"plaintext-data-key" def test_cmm_decrypt_materials_with_materials(self): """Test that DefaultCryptoMaterialsManager.decrypt_materials properly handles DecryptionMaterials input.""" @@ -143,13 +143,9 @@ def test_cmm_decrypt_materials_with_materials(self): result = cmm.decrypt_materials(materials) # Verify the result is a DecryptionMaterials instance - self.assertIsInstance(result, DecryptionMaterials) - self.assertEqual(result.iv, b"initialization-vector") - self.assertEqual(result.encrypted_data_keys, [edk]) - self.assertEqual(result.encryption_context_stored, {"key1": "value1"}) - self.assertEqual(result.encryption_context_from_request, {"key2": "value2"}) - self.assertEqual(result.plaintext_data_key, b"plaintext-data-key") - - -if __name__ == "__main__": - unittest.main() + assert isinstance(result, DecryptionMaterials) + assert result.iv == b"initialization-vector" + assert result.encrypted_data_keys == [edk] + assert result.encryption_context_stored == {"key1": "value1"} + assert result.encryption_context_from_request == {"key2": "value2"} + assert result.plaintext_data_key == b"plaintext-data-key" diff --git a/test/test_encryption_materials.py b/test/test_encryption_materials.py index ffa5a449..bd180d59 100644 --- a/test/test_encryption_materials.py +++ b/test/test_encryption_materials.py @@ -1,25 +1,25 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -import unittest +import pytest from src.s3_encryption.materials.encrypted_data_key import EncryptedDataKey from src.s3_encryption.materials.materials import EncryptionMaterials -class TestEncryptionMaterials(unittest.TestCase): +class TestEncryptionMaterials: def test_create_encryption_materials(self): """Test creating an EncryptionMaterials instance.""" materials = EncryptionMaterials() - self.assertEqual(materials.encryption_context, {}) - self.assertIsNone(materials.encrypted_data_key) - self.assertIsNone(materials.plaintext_data_key) + assert materials.encryption_context == {} + assert materials.encrypted_data_key is None + assert materials.plaintext_data_key is None def test_create_with_encryption_context(self): """Test creating an EncryptionMaterials instance with an encryption context.""" encryption_context = {"key1": "value1", "key2": "value2"} materials = EncryptionMaterials(encryption_context=encryption_context) - self.assertEqual(materials.encryption_context, encryption_context) + assert materials.encryption_context == encryption_context def test_from_dict(self): """Test creating an EncryptionMaterials instance from a dictionary.""" @@ -34,9 +34,9 @@ def test_from_dict(self): "PDK": b"plaintext-data-key", } materials = EncryptionMaterials.from_dict(materials_dict) - self.assertEqual(materials.encryption_context, {"key1": "value1"}) - self.assertEqual(materials.encrypted_data_key, edk) - self.assertEqual(materials.plaintext_data_key, b"plaintext-data-key") + assert materials.encryption_context == {"key1": "value1"} + assert materials.encrypted_data_key == edk + assert materials.plaintext_data_key == b"plaintext-data-key" def test_to_dict(self): """Test converting an EncryptionMaterials instance to a dictionary.""" @@ -51,10 +51,6 @@ def test_to_dict(self): plaintext_data_key=b"plaintext-data-key", ) materials_dict = materials.to_dict() - self.assertEqual(materials_dict["encryption_context"], {"key1": "value1"}) - self.assertEqual(materials_dict["encrypted_data_key"], edk) - self.assertEqual(materials_dict["PDK"], b"plaintext-data-key") - - -if __name__ == "__main__": - unittest.main() + assert materials_dict["encryption_context"] == {"key1": "value1"} + assert materials_dict["encrypted_data_key"] == edk + assert materials_dict["PDK"] == b"plaintext-data-key" diff --git a/test/test_encryption_materials_integration.py b/test/test_encryption_materials_integration.py index 28356fa8..abac6f8c 100644 --- a/test/test_encryption_materials_integration.py +++ b/test/test_encryption_materials_integration.py @@ -1,7 +1,7 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -import unittest +import pytest from unittest.mock import MagicMock from src.s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager @@ -10,7 +10,7 @@ from src.s3_encryption.materials.materials import EncryptionMaterials -class TestEncryptionMaterialsIntegration(unittest.TestCase): +class TestEncryptionMaterialsIntegration: def test_keyring_on_encrypt(self): """Test that S3Keyring.on_encrypt properly handles EncryptionMaterials.""" # Create a keyring @@ -23,8 +23,8 @@ def test_keyring_on_encrypt(self): result = keyring.on_encrypt(materials) # Verify the result is an EncryptionMaterials instance - self.assertIsInstance(result, EncryptionMaterials) - self.assertEqual(result.encryption_context, {"key1": "value1"}) + assert isinstance(result, EncryptionMaterials) + assert result.encryption_context == {"key1": "value1"} def test_cmm_get_encryption_materials_with_dict(self): """Test that DefaultCryptoMaterialsManager.get_encryption_materials properly handles dictionary input.""" @@ -47,10 +47,10 @@ def test_cmm_get_encryption_materials_with_dict(self): result = cmm.get_encryption_materials({"encryption_context": {"key1": "value1"}}) # Verify the result is an EncryptionMaterials instance - self.assertIsInstance(result, EncryptionMaterials) - self.assertEqual(result.encryption_context, {"key1": "value1"}) - self.assertIsNotNone(result.encrypted_data_key) - self.assertIsNotNone(result.plaintext_data_key) + assert isinstance(result, EncryptionMaterials) + assert result.encryption_context == {"key1": "value1"} + assert result.encrypted_data_key is not None + assert result.plaintext_data_key is not None def test_cmm_get_encryption_materials_with_materials(self): """Test that DefaultCryptoMaterialsManager.get_encryption_materials properly handles EncryptionMaterials input.""" @@ -74,11 +74,7 @@ def test_cmm_get_encryption_materials_with_materials(self): result = cmm.get_encryption_materials(materials) # Verify the result is an EncryptionMaterials instance - self.assertIsInstance(result, EncryptionMaterials) - self.assertEqual(result.encryption_context, {"key1": "value1"}) - self.assertIsNotNone(result.encrypted_data_key) - self.assertIsNotNone(result.plaintext_data_key) - - -if __name__ == "__main__": - unittest.main() + assert isinstance(result, EncryptionMaterials) + assert result.encryption_context == {"key1": "value1"} + assert result.encrypted_data_key is not None + assert result.plaintext_data_key is not None diff --git a/test/test_metadata.py b/test/test_metadata.py index 2c26c336..2abd2866 100644 --- a/test/test_metadata.py +++ b/test/test_metadata.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import os import sys -import unittest +import pytest # Add the src directory to the Python path sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../src"))) @@ -10,7 +10,7 @@ from s3_encryption.metadata import ObjectMetadata -class TestObjectMetadata(unittest.TestCase): +class TestObjectMetadata: def test_from_dict(self): # Create a metadata dictionary metadata_dict = { @@ -24,17 +24,17 @@ def test_from_dict(self): metadata = ObjectMetadata.from_dict(metadata_dict) # Verify that the fields were populated correctly - self.assertEqual(metadata.encrypted_data_key_v2, "encrypted-key-data") - self.assertEqual(metadata.encrypted_data_key_algorithm, "kms+context") - self.assertEqual(metadata.content_iv, "base64-encoded-iv") - self.assertEqual(metadata.content_cipher, "AES/GCM/NoPadding") + assert metadata.encrypted_data_key_v2 == "encrypted-key-data" + assert metadata.encrypted_data_key_algorithm == "kms+context" + assert metadata.content_iv == "base64-encoded-iv" + assert metadata.content_cipher == "AES/GCM/NoPadding" # Verify that fields not in the dictionary are None - self.assertIsNone(metadata.encrypted_data_key_v1) - self.assertIsNone(metadata.encrypted_data_key_context) + assert metadata.encrypted_data_key_v1 is None + assert metadata.encrypted_data_key_context is None # Note: content_cipher_tag_length is None because it's not in the input dictionary - self.assertIsNone(metadata.content_cipher_tag_length) - self.assertIsNone(metadata.instruction_file) + assert metadata.content_cipher_tag_length is None + assert metadata.instruction_file is None def test_to_dict(self): # Create an ObjectMetadata instance with some fields set @@ -49,17 +49,17 @@ def test_to_dict(self): metadata_dict = metadata.to_dict() # Verify that the dictionary contains the expected keys and values - self.assertEqual(metadata_dict["x-amz-key-v2"], "encrypted-key-data") - self.assertEqual(metadata_dict["x-amz-wrap-alg"], "kms+context") - self.assertEqual(metadata_dict["x-amz-iv"], "base64-encoded-iv") - self.assertEqual(metadata_dict["x-amz-cek-alg"], "AES/GCM/NoPadding") + assert metadata_dict["x-amz-key-v2"] == "encrypted-key-data" + assert metadata_dict["x-amz-wrap-alg"] == "kms+context" + assert metadata_dict["x-amz-iv"] == "base64-encoded-iv" + assert metadata_dict["x-amz-cek-alg"] == "AES/GCM/NoPadding" # Verify that fields that are None are not included in the dictionary - self.assertNotIn("x-amz-key", metadata_dict) - self.assertNotIn("x-amz-matdesc", metadata_dict) + assert "x-amz-key" not in metadata_dict + assert "x-amz-matdesc" not in metadata_dict # Note: content_cipher_tag_length has a default value of "128" - self.assertEqual(metadata_dict.get("x-amz-tag-len"), "128") - self.assertNotIn("x-amz-crypto-instr-file", metadata_dict) + assert metadata_dict.get("x-amz-tag-len") == "128" + assert "x-amz-crypto-instr-file" not in metadata_dict def test_roundtrip(self): # Create a metadata dictionary @@ -79,8 +79,4 @@ def test_roundtrip(self): result_dict.pop("x-amz-tag-len") # Verify that the result matches the original - self.assertEqual(result_dict, original_dict) - - -if __name__ == "__main__": - unittest.main() + assert result_dict == original_dict From 2f6bc1eefc2121f53e54eb8d60ddf73724cddc0e Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 11 Aug 2025 15:25:43 -0700 Subject: [PATCH 29/81] remove uv lock --- uv.lock | 369 -------------------------------------------------------- 1 file changed, 369 deletions(-) delete mode 100644 uv.lock diff --git a/uv.lock b/uv.lock deleted file mode 100644 index ebc7a7e9..00000000 --- a/uv.lock +++ /dev/null @@ -1,369 +0,0 @@ -version = 1 -revision = 3 -requires-python = ">=3.11" - -[[package]] -name = "amazon-s3-encryption-client-python" -version = "0.1.0" -source = { editable = "." } -dependencies = [ - { name = "attrs" }, - { name = "boto3" }, - { name = "cryptography" }, -] - -[package.optional-dependencies] -dev = [ - { name = "black" }, - { name = "ruff" }, -] -test = [ - { name = "pytest" }, -] - -[package.metadata] -requires-dist = [ - { name = "attrs", specifier = ">=25.1.0" }, - { name = "black", marker = "extra == 'dev'", specifier = ">=24.3.0" }, - { name = "boto3", specifier = ">=1.37.2" }, - { name = "cryptography", specifier = ">=45.0.6" }, - { name = "pytest", marker = "extra == 'test'", specifier = ">=8.4.1" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.3.0" }, -] -provides-extras = ["test", "dev"] - -[[package]] -name = "attrs" -version = "25.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/1367933a8532ee6ff8d63537de4f1177af4bff9f3e829baf7331f595bb24/attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b", size = 812032, upload-time = "2025-03-13T11:10:22.779Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, -] - -[[package]] -name = "black" -version = "25.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "mypy-extensions" }, - { name = "packaging" }, - { name = "pathspec" }, - { name = "platformdirs" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/94/49/26a7b0f3f35da4b5a65f081943b7bcd22d7002f5f0fb8098ec1ff21cb6ef/black-25.1.0.tar.gz", hash = "sha256:33496d5cd1222ad73391352b4ae8da15253c5de89b93a80b3e2c8d9a19ec2666", size = 649449, upload-time = "2025-01-29T04:15:40.373Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/4f/87f596aca05c3ce5b94b8663dbfe242a12843caaa82dd3f85f1ffdc3f177/black-25.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a39337598244de4bae26475f77dda852ea00a93bd4c728e09eacd827ec929df0", size = 1614372, upload-time = "2025-01-29T05:37:11.71Z" }, - { url = "https://files.pythonhosted.org/packages/e7/d0/2c34c36190b741c59c901e56ab7f6e54dad8df05a6272a9747ecef7c6036/black-25.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:96c1c7cd856bba8e20094e36e0f948718dc688dba4a9d78c3adde52b9e6c2299", size = 1442865, upload-time = "2025-01-29T05:37:14.309Z" }, - { url = "https://files.pythonhosted.org/packages/21/d4/7518c72262468430ead45cf22bd86c883a6448b9eb43672765d69a8f1248/black-25.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bce2e264d59c91e52d8000d507eb20a9aca4a778731a08cfff7e5ac4a4bb7096", size = 1749699, upload-time = "2025-01-29T04:18:17.688Z" }, - { url = "https://files.pythonhosted.org/packages/58/db/4f5beb989b547f79096e035c4981ceb36ac2b552d0ac5f2620e941501c99/black-25.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:172b1dbff09f86ce6f4eb8edf9dede08b1fce58ba194c87d7a4f1a5aa2f5b3c2", size = 1428028, upload-time = "2025-01-29T04:18:51.711Z" }, - { url = "https://files.pythonhosted.org/packages/83/71/3fe4741df7adf015ad8dfa082dd36c94ca86bb21f25608eb247b4afb15b2/black-25.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4b60580e829091e6f9238c848ea6750efed72140b91b048770b64e74fe04908b", size = 1650988, upload-time = "2025-01-29T05:37:16.707Z" }, - { url = "https://files.pythonhosted.org/packages/13/f3/89aac8a83d73937ccd39bbe8fc6ac8860c11cfa0af5b1c96d081facac844/black-25.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e2978f6df243b155ef5fa7e558a43037c3079093ed5d10fd84c43900f2d8ecc", size = 1453985, upload-time = "2025-01-29T05:37:18.273Z" }, - { url = "https://files.pythonhosted.org/packages/6f/22/b99efca33f1f3a1d2552c714b1e1b5ae92efac6c43e790ad539a163d1754/black-25.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b48735872ec535027d979e8dcb20bf4f70b5ac75a8ea99f127c106a7d7aba9f", size = 1783816, upload-time = "2025-01-29T04:18:33.823Z" }, - { url = "https://files.pythonhosted.org/packages/18/7e/a27c3ad3822b6f2e0e00d63d58ff6299a99a5b3aee69fa77cd4b0076b261/black-25.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:ea0213189960bda9cf99be5b8c8ce66bb054af5e9e861249cd23471bd7b0b3ba", size = 1440860, upload-time = "2025-01-29T04:19:12.944Z" }, - { url = "https://files.pythonhosted.org/packages/98/87/0edf98916640efa5d0696e1abb0a8357b52e69e82322628f25bf14d263d1/black-25.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f0b18a02996a836cc9c9c78e5babec10930862827b1b724ddfe98ccf2f2fe4f", size = 1650673, upload-time = "2025-01-29T05:37:20.574Z" }, - { url = "https://files.pythonhosted.org/packages/52/e5/f7bf17207cf87fa6e9b676576749c6b6ed0d70f179a3d812c997870291c3/black-25.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:afebb7098bfbc70037a053b91ae8437c3857482d3a690fefc03e9ff7aa9a5fd3", size = 1453190, upload-time = "2025-01-29T05:37:22.106Z" }, - { url = "https://files.pythonhosted.org/packages/e3/ee/adda3d46d4a9120772fae6de454c8495603c37c4c3b9c60f25b1ab6401fe/black-25.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:030b9759066a4ee5e5aca28c3c77f9c64789cdd4de8ac1df642c40b708be6171", size = 1782926, upload-time = "2025-01-29T04:18:58.564Z" }, - { url = "https://files.pythonhosted.org/packages/cc/64/94eb5f45dcb997d2082f097a3944cfc7fe87e071907f677e80788a2d7b7a/black-25.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:a22f402b410566e2d1c950708c77ebf5ebd5d0d88a6a2e87c86d9fb48afa0d18", size = 1442613, upload-time = "2025-01-29T04:19:27.63Z" }, - { url = "https://files.pythonhosted.org/packages/09/71/54e999902aed72baf26bca0d50781b01838251a462612966e9fc4891eadd/black-25.1.0-py3-none-any.whl", hash = "sha256:95e8176dae143ba9097f351d174fdaf0ccd29efb414b362ae3fd72bf0f710717", size = 207646, upload-time = "2025-01-29T04:15:38.082Z" }, -] - -[[package]] -name = "boto3" -version = "1.40.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, - { name = "jmespath" }, - { name = "s3transfer" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/58/5a/f31556d817e872c2723196a34b197d971d78297b22b8bae0ae6d93f7f9c1/boto3-1.40.7.tar.gz", hash = "sha256:61b15f70761f1eadd721c6ba41a92658f003eaaef09500ca7642f5ae68ec8945", size = 111989, upload-time = "2025-08-11T19:20:45.824Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/e3/f2a77f4809ffe4e896c2e6186db88333ae980f52a91b28e9fd068d8f5506/boto3-1.40.7-py3-none-any.whl", hash = "sha256:8727cac601a679d2885dc78b8119a0548bbbe04e49b72f7d94021a629154c080", size = 140061, upload-time = "2025-08-11T19:20:43.173Z" }, -] - -[[package]] -name = "botocore" -version = "1.40.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jmespath" }, - { name = "python-dateutil" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/73/d7/5e559918410b259c1e54a4646ff39c56433e1c9cefa5e66ab0f06716cee8/botocore-1.40.7.tar.gz", hash = "sha256:33793696680cf3a0c4b5ace4f9070c67c4d4fcb19c999fd85cfee55de3dcf913", size = 14318282, upload-time = "2025-08-11T19:20:33.348Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/fa/bb7ec68b24d1b4678d341a305cbfed78a593e6383c86a70727410e4d0e11/botocore-1.40.7-py3-none-any.whl", hash = "sha256:a06956f3d7222e80ef6ae193608f358c3b7898e1a2b88553479d8f9737fbb03e", size = 13981488, upload-time = "2025-08-11T19:20:27.303Z" }, -] - -[[package]] -name = "cffi" -version = "1.17.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pycparser" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload-time = "2024-09-04T20:45:21.852Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264, upload-time = "2024-09-04T20:43:51.124Z" }, - { url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651, upload-time = "2024-09-04T20:43:52.872Z" }, - { url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259, upload-time = "2024-09-04T20:43:56.123Z" }, - { url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200, upload-time = "2024-09-04T20:43:57.891Z" }, - { url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235, upload-time = "2024-09-04T20:44:00.18Z" }, - { url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721, upload-time = "2024-09-04T20:44:01.585Z" }, - { url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242, upload-time = "2024-09-04T20:44:03.467Z" }, - { url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999, upload-time = "2024-09-04T20:44:05.023Z" }, - { url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242, upload-time = "2024-09-04T20:44:06.444Z" }, - { url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604, upload-time = "2024-09-04T20:44:08.206Z" }, - { url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727, upload-time = "2024-09-04T20:44:09.481Z" }, - { url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400, upload-time = "2024-09-04T20:44:10.873Z" }, - { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178, upload-time = "2024-09-04T20:44:12.232Z" }, - { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840, upload-time = "2024-09-04T20:44:13.739Z" }, - { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803, upload-time = "2024-09-04T20:44:15.231Z" }, - { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850, upload-time = "2024-09-04T20:44:17.188Z" }, - { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729, upload-time = "2024-09-04T20:44:18.688Z" }, - { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256, upload-time = "2024-09-04T20:44:20.248Z" }, - { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424, upload-time = "2024-09-04T20:44:21.673Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568, upload-time = "2024-09-04T20:44:23.245Z" }, - { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736, upload-time = "2024-09-04T20:44:24.757Z" }, - { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448, upload-time = "2024-09-04T20:44:26.208Z" }, - { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976, upload-time = "2024-09-04T20:44:27.578Z" }, - { url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989, upload-time = "2024-09-04T20:44:28.956Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802, upload-time = "2024-09-04T20:44:30.289Z" }, - { url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792, upload-time = "2024-09-04T20:44:32.01Z" }, - { url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893, upload-time = "2024-09-04T20:44:33.606Z" }, - { url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810, upload-time = "2024-09-04T20:44:35.191Z" }, - { url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200, upload-time = "2024-09-04T20:44:36.743Z" }, - { url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447, upload-time = "2024-09-04T20:44:38.492Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358, upload-time = "2024-09-04T20:44:40.046Z" }, - { url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469, upload-time = "2024-09-04T20:44:41.616Z" }, - { url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475, upload-time = "2024-09-04T20:44:43.733Z" }, - { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009, upload-time = "2024-09-04T20:44:45.309Z" }, -] - -[[package]] -name = "click" -version = "8.2.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/60/6c/8ca2efa64cf75a977a0d7fac081354553ebe483345c734fb6b6515d96bbc/click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202", size = 286342, upload-time = "2025-05-20T23:19:49.832Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215, upload-time = "2025-05-20T23:19:47.796Z" }, -] - -[[package]] -name = "colorama" -version = "0.4.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, -] - -[[package]] -name = "cryptography" -version = "45.0.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d6/0d/d13399c94234ee8f3df384819dc67e0c5ce215fb751d567a55a1f4b028c7/cryptography-45.0.6.tar.gz", hash = "sha256:5c966c732cf6e4a276ce83b6e4c729edda2df6929083a952cc7da973c539c719", size = 744949, upload-time = "2025-08-05T23:59:27.93Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8c/29/2793d178d0eda1ca4a09a7c4e09a5185e75738cc6d526433e8663b460ea6/cryptography-45.0.6-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:048e7ad9e08cf4c0ab07ff7f36cc3115924e22e2266e034450a890d9e312dd74", size = 7042702, upload-time = "2025-08-05T23:58:23.464Z" }, - { url = "https://files.pythonhosted.org/packages/b3/b6/cabd07410f222f32c8d55486c464f432808abaa1f12af9afcbe8f2f19030/cryptography-45.0.6-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:44647c5d796f5fc042bbc6d61307d04bf29bccb74d188f18051b635f20a9c75f", size = 4206483, upload-time = "2025-08-05T23:58:27.132Z" }, - { url = "https://files.pythonhosted.org/packages/8b/9e/f9c7d36a38b1cfeb1cc74849aabe9bf817990f7603ff6eb485e0d70e0b27/cryptography-45.0.6-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e40b80ecf35ec265c452eea0ba94c9587ca763e739b8e559c128d23bff7ebbbf", size = 4429679, upload-time = "2025-08-05T23:58:29.152Z" }, - { url = "https://files.pythonhosted.org/packages/9c/2a/4434c17eb32ef30b254b9e8b9830cee4e516f08b47fdd291c5b1255b8101/cryptography-45.0.6-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:00e8724bdad672d75e6f069b27970883179bd472cd24a63f6e620ca7e41cc0c5", size = 4210553, upload-time = "2025-08-05T23:58:30.596Z" }, - { url = "https://files.pythonhosted.org/packages/ef/1d/09a5df8e0c4b7970f5d1f3aff1b640df6d4be28a64cae970d56c6cf1c772/cryptography-45.0.6-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7a3085d1b319d35296176af31c90338eeb2ddac8104661df79f80e1d9787b8b2", size = 3894499, upload-time = "2025-08-05T23:58:32.03Z" }, - { url = "https://files.pythonhosted.org/packages/79/62/120842ab20d9150a9d3a6bdc07fe2870384e82f5266d41c53b08a3a96b34/cryptography-45.0.6-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1b7fa6a1c1188c7ee32e47590d16a5a0646270921f8020efc9a511648e1b2e08", size = 4458484, upload-time = "2025-08-05T23:58:33.526Z" }, - { url = "https://files.pythonhosted.org/packages/fd/80/1bc3634d45ddfed0871bfba52cf8f1ad724761662a0c792b97a951fb1b30/cryptography-45.0.6-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:275ba5cc0d9e320cd70f8e7b96d9e59903c815ca579ab96c1e37278d231fc402", size = 4210281, upload-time = "2025-08-05T23:58:35.445Z" }, - { url = "https://files.pythonhosted.org/packages/7d/fe/ffb12c2d83d0ee625f124880a1f023b5878f79da92e64c37962bbbe35f3f/cryptography-45.0.6-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:f4028f29a9f38a2025abedb2e409973709c660d44319c61762202206ed577c42", size = 4456890, upload-time = "2025-08-05T23:58:36.923Z" }, - { url = "https://files.pythonhosted.org/packages/8c/8e/b3f3fe0dc82c77a0deb5f493b23311e09193f2268b77196ec0f7a36e3f3e/cryptography-45.0.6-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ee411a1b977f40bd075392c80c10b58025ee5c6b47a822a33c1198598a7a5f05", size = 4333247, upload-time = "2025-08-05T23:58:38.781Z" }, - { url = "https://files.pythonhosted.org/packages/b3/a6/c3ef2ab9e334da27a1d7b56af4a2417d77e7806b2e0f90d6267ce120d2e4/cryptography-45.0.6-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e2a21a8eda2d86bb604934b6b37691585bd095c1f788530c1fcefc53a82b3453", size = 4565045, upload-time = "2025-08-05T23:58:40.415Z" }, - { url = "https://files.pythonhosted.org/packages/31/c3/77722446b13fa71dddd820a5faab4ce6db49e7e0bf8312ef4192a3f78e2f/cryptography-45.0.6-cp311-abi3-win32.whl", hash = "sha256:d063341378d7ee9c91f9d23b431a3502fc8bfacd54ef0a27baa72a0843b29159", size = 2928923, upload-time = "2025-08-05T23:58:41.919Z" }, - { url = "https://files.pythonhosted.org/packages/38/63/a025c3225188a811b82932a4dcc8457a26c3729d81578ccecbcce2cb784e/cryptography-45.0.6-cp311-abi3-win_amd64.whl", hash = "sha256:833dc32dfc1e39b7376a87b9a6a4288a10aae234631268486558920029b086ec", size = 3403805, upload-time = "2025-08-05T23:58:43.792Z" }, - { url = "https://files.pythonhosted.org/packages/5b/af/bcfbea93a30809f126d51c074ee0fac5bd9d57d068edf56c2a73abedbea4/cryptography-45.0.6-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:3436128a60a5e5490603ab2adbabc8763613f638513ffa7d311c900a8349a2a0", size = 7020111, upload-time = "2025-08-05T23:58:45.316Z" }, - { url = "https://files.pythonhosted.org/packages/98/c6/ea5173689e014f1a8470899cd5beeb358e22bb3cf5a876060f9d1ca78af4/cryptography-45.0.6-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0d9ef57b6768d9fa58e92f4947cea96ade1233c0e236db22ba44748ffedca394", size = 4198169, upload-time = "2025-08-05T23:58:47.121Z" }, - { url = "https://files.pythonhosted.org/packages/ba/73/b12995edc0c7e2311ffb57ebd3b351f6b268fed37d93bfc6f9856e01c473/cryptography-45.0.6-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea3c42f2016a5bbf71825537c2ad753f2870191134933196bee408aac397b3d9", size = 4421273, upload-time = "2025-08-05T23:58:48.557Z" }, - { url = "https://files.pythonhosted.org/packages/f7/6e/286894f6f71926bc0da67408c853dd9ba953f662dcb70993a59fd499f111/cryptography-45.0.6-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:20ae4906a13716139d6d762ceb3e0e7e110f7955f3bc3876e3a07f5daadec5f3", size = 4199211, upload-time = "2025-08-05T23:58:50.139Z" }, - { url = "https://files.pythonhosted.org/packages/de/34/a7f55e39b9623c5cb571d77a6a90387fe557908ffc44f6872f26ca8ae270/cryptography-45.0.6-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dac5ec199038b8e131365e2324c03d20e97fe214af051d20c49db129844e8b3", size = 3883732, upload-time = "2025-08-05T23:58:52.253Z" }, - { url = "https://files.pythonhosted.org/packages/f9/b9/c6d32edbcba0cd9f5df90f29ed46a65c4631c4fbe11187feb9169c6ff506/cryptography-45.0.6-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:18f878a34b90d688982e43f4b700408b478102dd58b3e39de21b5ebf6509c301", size = 4450655, upload-time = "2025-08-05T23:58:53.848Z" }, - { url = "https://files.pythonhosted.org/packages/77/2d/09b097adfdee0227cfd4c699b3375a842080f065bab9014248933497c3f9/cryptography-45.0.6-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5bd6020c80c5b2b2242d6c48487d7b85700f5e0038e67b29d706f98440d66eb5", size = 4198956, upload-time = "2025-08-05T23:58:55.209Z" }, - { url = "https://files.pythonhosted.org/packages/55/66/061ec6689207d54effdff535bbdf85cc380d32dd5377173085812565cf38/cryptography-45.0.6-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:eccddbd986e43014263eda489abbddfbc287af5cddfd690477993dbb31e31016", size = 4449859, upload-time = "2025-08-05T23:58:56.639Z" }, - { url = "https://files.pythonhosted.org/packages/41/ff/e7d5a2ad2d035e5a2af116e1a3adb4d8fcd0be92a18032917a089c6e5028/cryptography-45.0.6-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:550ae02148206beb722cfe4ef0933f9352bab26b087af00e48fdfb9ade35c5b3", size = 4320254, upload-time = "2025-08-05T23:58:58.833Z" }, - { url = "https://files.pythonhosted.org/packages/82/27/092d311af22095d288f4db89fcaebadfb2f28944f3d790a4cf51fe5ddaeb/cryptography-45.0.6-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5b64e668fc3528e77efa51ca70fadcd6610e8ab231e3e06ae2bab3b31c2b8ed9", size = 4554815, upload-time = "2025-08-05T23:59:00.283Z" }, - { url = "https://files.pythonhosted.org/packages/7e/01/aa2f4940262d588a8fdf4edabe4cda45854d00ebc6eaac12568b3a491a16/cryptography-45.0.6-cp37-abi3-win32.whl", hash = "sha256:780c40fb751c7d2b0c6786ceee6b6f871e86e8718a8ff4bc35073ac353c7cd02", size = 2912147, upload-time = "2025-08-05T23:59:01.716Z" }, - { url = "https://files.pythonhosted.org/packages/0a/bc/16e0276078c2de3ceef6b5a34b965f4436215efac45313df90d55f0ba2d2/cryptography-45.0.6-cp37-abi3-win_amd64.whl", hash = "sha256:20d15aed3ee522faac1a39fbfdfee25d17b1284bafd808e1640a74846d7c4d1b", size = 3390459, upload-time = "2025-08-05T23:59:03.358Z" }, - { url = "https://files.pythonhosted.org/packages/61/69/c252de4ec047ba2f567ecb53149410219577d408c2aea9c989acae7eafce/cryptography-45.0.6-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:fc022c1fa5acff6def2fc6d7819bbbd31ccddfe67d075331a65d9cfb28a20983", size = 3584669, upload-time = "2025-08-05T23:59:15.431Z" }, - { url = "https://files.pythonhosted.org/packages/e3/fe/deea71e9f310a31fe0a6bfee670955152128d309ea2d1c79e2a5ae0f0401/cryptography-45.0.6-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:3de77e4df42ac8d4e4d6cdb342d989803ad37707cf8f3fbf7b088c9cbdd46427", size = 4153022, upload-time = "2025-08-05T23:59:16.954Z" }, - { url = "https://files.pythonhosted.org/packages/60/45/a77452f5e49cb580feedba6606d66ae7b82c128947aa754533b3d1bd44b0/cryptography-45.0.6-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:599c8d7df950aa68baa7e98f7b73f4f414c9f02d0e8104a30c0182a07732638b", size = 4386802, upload-time = "2025-08-05T23:59:18.55Z" }, - { url = "https://files.pythonhosted.org/packages/a3/b9/a2f747d2acd5e3075fdf5c145c7c3568895daaa38b3b0c960ef830db6cdc/cryptography-45.0.6-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:31a2b9a10530a1cb04ffd6aa1cd4d3be9ed49f7d77a4dafe198f3b382f41545c", size = 4152706, upload-time = "2025-08-05T23:59:20.044Z" }, - { url = "https://files.pythonhosted.org/packages/81/ec/381b3e8d0685a3f3f304a382aa3dfce36af2d76467da0fd4bb21ddccc7b2/cryptography-45.0.6-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:e5b3dda1b00fb41da3af4c5ef3f922a200e33ee5ba0f0bc9ecf0b0c173958385", size = 4386740, upload-time = "2025-08-05T23:59:21.525Z" }, - { url = "https://files.pythonhosted.org/packages/0a/76/cf8d69da8d0b5ecb0db406f24a63a3f69ba5e791a11b782aeeefef27ccbb/cryptography-45.0.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:629127cfdcdc6806dfe234734d7cb8ac54edaf572148274fa377a7d3405b0043", size = 3331874, upload-time = "2025-08-05T23:59:23.017Z" }, -] - -[[package]] -name = "iniconfig" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, -] - -[[package]] -name = "jmespath" -version = "1.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" }, -] - -[[package]] -name = "mypy-extensions" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, -] - -[[package]] -name = "packaging" -version = "25.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, -] - -[[package]] -name = "pathspec" -version = "0.12.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, -] - -[[package]] -name = "platformdirs" -version = "4.3.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fe/8b/3c73abc9c759ecd3f1f7ceff6685840859e8070c4d947c93fae71f6a0bf2/platformdirs-4.3.8.tar.gz", hash = "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc", size = 21362, upload-time = "2025-05-07T22:47:42.121Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/39/979e8e21520d4e47a0bbe349e2713c0aac6f3d853d0e5b34d76206c439aa/platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4", size = 18567, upload-time = "2025-05-07T22:47:40.376Z" }, -] - -[[package]] -name = "pluggy" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, -] - -[[package]] -name = "pycparser" -version = "2.22" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736, upload-time = "2024-03-30T13:22:22.564Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552, upload-time = "2024-03-30T13:22:20.476Z" }, -] - -[[package]] -name = "pygments" -version = "2.19.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, -] - -[[package]] -name = "pytest" -version = "8.4.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "iniconfig" }, - { name = "packaging" }, - { name = "pluggy" }, - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/08/ba/45911d754e8eba3d5a841a5ce61a65a685ff1798421ac054f85aa8747dfb/pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c", size = 1517714, upload-time = "2025-06-18T05:48:06.109Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474, upload-time = "2025-06-18T05:48:03.955Z" }, -] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, -] - -[[package]] -name = "ruff" -version = "0.12.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4b/da/5bd7565be729e86e1442dad2c9a364ceeff82227c2dece7c29697a9795eb/ruff-0.12.8.tar.gz", hash = "sha256:4cb3a45525176e1009b2b64126acf5f9444ea59066262791febf55e40493a033", size = 5242373, upload-time = "2025-08-07T19:05:47.268Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/1e/c843bfa8ad1114fab3eb2b78235dda76acd66384c663a4e0415ecc13aa1e/ruff-0.12.8-py3-none-linux_armv6l.whl", hash = "sha256:63cb5a5e933fc913e5823a0dfdc3c99add73f52d139d6cd5cc8639d0e0465513", size = 11675315, upload-time = "2025-08-07T19:05:06.15Z" }, - { url = "https://files.pythonhosted.org/packages/24/ee/af6e5c2a8ca3a81676d5480a1025494fd104b8896266502bb4de2a0e8388/ruff-0.12.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9a9bbe28f9f551accf84a24c366c1aa8774d6748438b47174f8e8565ab9dedbc", size = 12456653, upload-time = "2025-08-07T19:05:09.759Z" }, - { url = "https://files.pythonhosted.org/packages/99/9d/e91f84dfe3866fa648c10512904991ecc326fd0b66578b324ee6ecb8f725/ruff-0.12.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:2fae54e752a3150f7ee0e09bce2e133caf10ce9d971510a9b925392dc98d2fec", size = 11659690, upload-time = "2025-08-07T19:05:12.551Z" }, - { url = "https://files.pythonhosted.org/packages/fe/ac/a363d25ec53040408ebdd4efcee929d48547665858ede0505d1d8041b2e5/ruff-0.12.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0acbcf01206df963d9331b5838fb31f3b44fa979ee7fa368b9b9057d89f4a53", size = 11896923, upload-time = "2025-08-07T19:05:14.821Z" }, - { url = "https://files.pythonhosted.org/packages/58/9f/ea356cd87c395f6ade9bb81365bd909ff60860975ca1bc39f0e59de3da37/ruff-0.12.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ae3e7504666ad4c62f9ac8eedb52a93f9ebdeb34742b8b71cd3cccd24912719f", size = 11477612, upload-time = "2025-08-07T19:05:16.712Z" }, - { url = "https://files.pythonhosted.org/packages/1a/46/92e8fa3c9dcfd49175225c09053916cb97bb7204f9f899c2f2baca69e450/ruff-0.12.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb82efb5d35d07497813a1c5647867390a7d83304562607f3579602fa3d7d46f", size = 13182745, upload-time = "2025-08-07T19:05:18.709Z" }, - { url = "https://files.pythonhosted.org/packages/5e/c4/f2176a310f26e6160deaf661ef60db6c3bb62b7a35e57ae28f27a09a7d63/ruff-0.12.8-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:dbea798fc0065ad0b84a2947b0aff4233f0cb30f226f00a2c5850ca4393de609", size = 14206885, upload-time = "2025-08-07T19:05:21.025Z" }, - { url = "https://files.pythonhosted.org/packages/87/9d/98e162f3eeeb6689acbedbae5050b4b3220754554526c50c292b611d3a63/ruff-0.12.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:49ebcaccc2bdad86fd51b7864e3d808aad404aab8df33d469b6e65584656263a", size = 13639381, upload-time = "2025-08-07T19:05:23.423Z" }, - { url = "https://files.pythonhosted.org/packages/81/4e/1b7478b072fcde5161b48f64774d6edd59d6d198e4ba8918d9f4702b8043/ruff-0.12.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ac9c570634b98c71c88cb17badd90f13fc076a472ba6ef1d113d8ed3df109fb", size = 12613271, upload-time = "2025-08-07T19:05:25.507Z" }, - { url = "https://files.pythonhosted.org/packages/e8/67/0c3c9179a3ad19791ef1b8f7138aa27d4578c78700551c60d9260b2c660d/ruff-0.12.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:560e0cd641e45591a3e42cb50ef61ce07162b9c233786663fdce2d8557d99818", size = 12847783, upload-time = "2025-08-07T19:05:28.14Z" }, - { url = "https://files.pythonhosted.org/packages/4e/2a/0b6ac3dd045acf8aa229b12c9c17bb35508191b71a14904baf99573a21bd/ruff-0.12.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:71c83121512e7743fba5a8848c261dcc454cafb3ef2934a43f1b7a4eb5a447ea", size = 11702672, upload-time = "2025-08-07T19:05:30.413Z" }, - { url = "https://files.pythonhosted.org/packages/9d/ee/f9fdc9f341b0430110de8b39a6ee5fa68c5706dc7c0aa940817947d6937e/ruff-0.12.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:de4429ef2ba091ecddedd300f4c3f24bca875d3d8b23340728c3cb0da81072c3", size = 11440626, upload-time = "2025-08-07T19:05:32.492Z" }, - { url = "https://files.pythonhosted.org/packages/89/fb/b3aa2d482d05f44e4d197d1de5e3863feb13067b22c571b9561085c999dc/ruff-0.12.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a2cab5f60d5b65b50fba39a8950c8746df1627d54ba1197f970763917184b161", size = 12462162, upload-time = "2025-08-07T19:05:34.449Z" }, - { url = "https://files.pythonhosted.org/packages/18/9f/5c5d93e1d00d854d5013c96e1a92c33b703a0332707a7cdbd0a4880a84fb/ruff-0.12.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:45c32487e14f60b88aad6be9fd5da5093dbefb0e3e1224131cb1d441d7cb7d46", size = 12913212, upload-time = "2025-08-07T19:05:36.541Z" }, - { url = "https://files.pythonhosted.org/packages/71/13/ab9120add1c0e4604c71bfc2e4ef7d63bebece0cfe617013da289539cef8/ruff-0.12.8-py3-none-win32.whl", hash = "sha256:daf3475060a617fd5bc80638aeaf2f5937f10af3ec44464e280a9d2218e720d3", size = 11694382, upload-time = "2025-08-07T19:05:38.468Z" }, - { url = "https://files.pythonhosted.org/packages/f6/dc/a2873b7c5001c62f46266685863bee2888caf469d1edac84bf3242074be2/ruff-0.12.8-py3-none-win_amd64.whl", hash = "sha256:7209531f1a1fcfbe8e46bcd7ab30e2f43604d8ba1c49029bb420b103d0b5f76e", size = 12740482, upload-time = "2025-08-07T19:05:40.391Z" }, - { url = "https://files.pythonhosted.org/packages/cb/5c/799a1efb8b5abab56e8a9f2a0b72d12bd64bb55815e9476c7d0a2887d2f7/ruff-0.12.8-py3-none-win_arm64.whl", hash = "sha256:c90e1a334683ce41b0e7a04f41790c429bf5073b62c1ae701c9dc5b3d14f0749", size = 11884718, upload-time = "2025-08-07T19:05:42.866Z" }, -] - -[[package]] -name = "s3transfer" -version = "0.13.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6d/05/d52bf1e65044b4e5e27d4e63e8d1579dbdec54fce685908ae09bc3720030/s3transfer-0.13.1.tar.gz", hash = "sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf", size = 150589, upload-time = "2025-07-18T19:22:42.31Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/4f/d073e09df851cfa251ef7840007d04db3293a0482ce607d2b993926089be/s3transfer-0.13.1-py3-none-any.whl", hash = "sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724", size = 85308, upload-time = "2025-07-18T19:22:40.947Z" }, -] - -[[package]] -name = "six" -version = "1.17.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, -] - -[[package]] -name = "urllib3" -version = "2.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, -] From fa000a5983c940b3df38d555bf7d680ba48dd1c7 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Tue, 12 Aug 2025 09:52:30 -0700 Subject: [PATCH 30/81] ruff fixes --- src/s3_encryption/__init__.py | 37 +++++++++++++++++-- src/s3_encryption/exceptions.py | 4 ++ src/s3_encryption/materials/__init__.py | 5 +++ .../materials/crypto_materials_manager.py | 26 +++++++++++-- .../materials/encrypted_data_key.py | 5 +++ src/s3_encryption/materials/keyring.py | 29 +++++++++------ src/s3_encryption/materials/kms_keyring.py | 30 ++++++++++++--- src/s3_encryption/materials/materials.py | 6 +++ src/s3_encryption/metadata.py | 8 +++- src/s3_encryption/pipelines.py | 17 ++++++--- 10 files changed, 137 insertions(+), 30 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 0b56454b..31ccf0a6 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -1,5 +1,6 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +"""Top-level S3 Encryption Client v3 for Python package.""" import io from attrs import define, field @@ -10,13 +11,12 @@ DefaultCryptoMaterialsManager, ) from .materials.keyring import AbstractKeyring -from .metadata import ObjectMetadata from .pipelines import GetEncryptedObjectPipeline, PutEncryptedObjectPipeline @define class S3EncryptionClientConfig: - """Configuration object for the S3 Encryption Client""" + """Configuration object for the S3 Encryption Client.""" keyring: AbstractKeyring cmm: AbstractCryptoMaterialsManager = field() @@ -28,10 +28,28 @@ def _default_cmm_for_keyring(self): @define class S3EncryptionClient: + """Client for encrypting and decrypting S3 objects. + + This client wraps a boto3 S3 client and provides encryption and decryption + capabilities for S3 objects using the configured keyring and crypto materials manager. + """ wrapped_s3_client = field() config: S3EncryptionClientConfig = field() def put_object(self, **kwargs): + """Encrypt and upload an object to S3. + + This method encrypts the provided object body before uploading it to S3. + It handles the encryption process using the configured crypto materials manager. + + Args: + **kwargs: Arguments to pass to the S3 client's put_object method. + Must include Bucket, Key, and Body parameters. + May include EncryptionContext for additional authenticated data. + + Returns: + The response from the S3 client's put_object method. + """ # Extract required parameters from kwargs bucket = kwargs.pop("Bucket") key = kwargs.pop("Key") @@ -45,7 +63,7 @@ def put_object(self, **kwargs): data_bytes = body # We probably just shouldn't support strings, use utf8 for now # TODO: look deeper into this, what does normal boto3 do? - if type(body) == str: + if isinstance(body, str): data_bytes = body.encode("utf-8") encrypted_data, encryption_metadata = pipeline.encrypt( data_bytes, encryption_context=encryption_context @@ -64,6 +82,19 @@ def put_object(self, **kwargs): return self.wrapped_s3_client.put_object(**params) def get_object(self, **kwargs): + """Download and decrypt an object from S3. + + This method downloads an encrypted object from S3 and decrypts it + using the configured crypto materials manager. + + Args: + **kwargs: Arguments to pass to the S3 client's get_object method. + May include EncryptionContext if it was used during encryption. + + Returns: + The response from the S3 client's get_object method with the Body + replaced with a StreamingBody containing the decrypted data. + """ # Extract encryption context if provided encryption_context = kwargs.pop("EncryptionContext", None) diff --git a/src/s3_encryption/exceptions.py b/src/s3_encryption/exceptions.py index 034b2bdf..d7d44df2 100644 --- a/src/s3_encryption/exceptions.py +++ b/src/s3_encryption/exceptions.py @@ -1,4 +1,8 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +"""Exceptions for the S3 Encryption Client. + +This module contains custom exception classes used throughout the S3 Encryption Client. +""" class S3EncryptionClientError(Exception): """Exception class for S3 Encryption Client errors.""" diff --git a/src/s3_encryption/materials/__init__.py b/src/s3_encryption/materials/__init__.py index b602bc91..c67d5802 100644 --- a/src/s3_encryption/materials/__init__.py +++ b/src/s3_encryption/materials/__init__.py @@ -1,5 +1,10 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +"""Materials package for S3 Encryption Client. + +This package contains classes and interfaces for cryptographic materials +management, including keyrings, crypto materials managers, and encrypted data keys. +""" from .crypto_materials_manager import AbstractCryptoMaterialsManager, DefaultCryptoMaterialsManager from .encrypted_data_key import EncryptedDataKey diff --git a/src/s3_encryption/materials/crypto_materials_manager.py b/src/s3_encryption/materials/crypto_materials_manager.py index 06965eff..3ecb2489 100644 --- a/src/s3_encryption/materials/crypto_materials_manager.py +++ b/src/s3_encryption/materials/crypto_materials_manager.py @@ -1,5 +1,10 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +"""Crypto materials manager module for S3 Encryption Client. + +This module provides interfaces and implementations for crypto materials managers, +which are responsible for coordinating the generation and use of cryptographic materials. +""" import abc @@ -11,12 +16,18 @@ # API Stub for CMM class AbstractCryptoMaterialsManager(abc.ABC): + """Abstract base class for crypto materials managers. + + A crypto materials manager is responsible for generating encryption materials + and processing decryption materials using a keyring. + """ @abc.abstractmethod def get_encryption_materials(self, enc_mats_request): """Get encryption materials from the keyring. Args: - enc_mats_request (Dict[str, Any] or EncryptionMaterials): Request containing encryption parameters + enc_mats_request (Dict[str, Any] or EncryptionMaterials): Request containing encryption + parameters Returns: EncryptionMaterials: The encryption materials @@ -28,7 +39,8 @@ def decrypt_materials(self, dec_mats_request): """Decrypt materials using the keyring. Args: - dec_mats_request (Dict[str, Any] or DecryptionMaterials): Request containing decryption parameters + dec_mats_request (Dict[str, Any] or DecryptionMaterials): Request containing decryption + parameters Returns: DecryptionMaterials: The decryption materials @@ -38,6 +50,13 @@ def decrypt_materials(self, dec_mats_request): @define class DefaultCryptoMaterialsManager(AbstractCryptoMaterialsManager): + """Default implementation of the crypto materials manager. + + This implementation delegates encryption and decryption operations to a single keyring. + + Attributes: + keyring (AbstractKeyring): The keyring to use for cryptographic operations + """ keyring: AbstractKeyring def get_encryption_materials(self, enc_mats_request): @@ -63,7 +82,8 @@ def decrypt_materials(self, dec_mats_request): """Decrypt materials using the keyring. Args: - dec_mats_request (Dict[str, Any] or DecryptionMaterials): Request containing decryption parameters + dec_mats_request (Dict[str, Any] or DecryptionMaterials): Request containing decryption + parameters Returns: DecryptionMaterials: The decryption materials diff --git a/src/s3_encryption/materials/encrypted_data_key.py b/src/s3_encryption/materials/encrypted_data_key.py index 057fafda..28401b40 100644 --- a/src/s3_encryption/materials/encrypted_data_key.py +++ b/src/s3_encryption/materials/encrypted_data_key.py @@ -1,5 +1,10 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +"""Encrypted data key module for S3 Encryption Client. + +This module provides the EncryptedDataKey class which represents an encrypted +data key used in the S3 encryption process. +""" from attrs import define diff --git a/src/s3_encryption/materials/keyring.py b/src/s3_encryption/materials/keyring.py index 1f1e5e83..6115b02f 100644 --- a/src/s3_encryption/materials/keyring.py +++ b/src/s3_encryption/materials/keyring.py @@ -1,6 +1,10 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +"""Keyring module for S3 Encryption Client. +This module provides interfaces and implementations for keyrings, which are +responsible for encrypting and decrypting data keys used in the S3 encryption process. +""" import abc @@ -12,11 +16,11 @@ @define class AbstractKeyring(abc.ABC): - # Ideally, all keyrings would inherit this field. - # However, attrs doesn't allow us to set a default here, - # when inheriting keyrings have optional fields. - # Even without a default it doesn't seem to play nice with attrs. - # enableLegacyWrappingAlgorithms: bool = field(default=False) + """Abstract base class for keyrings. + + A keyring is responsible for encrypting and decrypting data keys. + Concrete implementations handle specific key providers like KMS. + """ @abc.abstractmethod def on_encrypt(self, enc_materials): @@ -35,8 +39,10 @@ def on_decrypt(self, dec_materials, encrypted_data_keys=None): """Decrypt one of the encrypted data keys and update dec_materials. Args: - dec_materials (DecryptionMaterials): A DecryptionMaterials instance containing decryption materials - encrypted_data_keys (List[EncryptedDataKey], optional): A list of encrypted data keys to try. + dec_materials (DecryptionMaterials): A DecryptionMaterials instance containing + decryption materials + encrypted_data_keys (List[EncryptedDataKey], optional): A list of encrypted data + keys to try. Returns: DecryptionMaterials: The updated dec_materials with the plaintext data key (PDK) @@ -48,9 +54,6 @@ def on_decrypt(self, dec_materials, encrypted_data_keys=None): class S3Keyring(AbstractKeyring): """Base class for S3 encryption keyrings that provides common validation logic.""" - # Ideally this would be set, but attrs doesn't play nice - # enable_legacy_wrapping_algorithms: bool = field(default=False) - def on_encrypt(self, enc_materials): """Validate encryption materials before encryption. @@ -80,8 +83,10 @@ def on_decrypt(self, dec_materials, encrypted_data_keys=None): """Validate decryption materials before decryption. Args: - dec_materials (DecryptionMaterials): A DecryptionMaterials instance containing decryption materials - encrypted_data_keys (List[EncryptedDataKey], optional): A list of encrypted data keys to try. + dec_materials (DecryptionMaterials): A DecryptionMaterials instance containing + decryption materials + encrypted_data_keys (List[EncryptedDataKey], optional): A list of encrypted data + keys to try. Returns: DecryptionMaterials: The validated decryption materials diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index 8b6914da..303a5f93 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -1,8 +1,12 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +"""KMS keyring module for S3 Encryption Client. + +This module provides a KMS-based keyring implementation that uses AWS KMS +to generate and decrypt data keys for S3 object encryption. +""" from attrs import define, field -from botocore import client from ..exceptions import S3EncryptionClientError from .encrypted_data_key import EncryptedDataKey @@ -14,6 +18,15 @@ @define class KmsKeyring(S3Keyring): + """KMS implementation of the S3 keyring. + + This keyring uses AWS KMS to generate and decrypt data keys. + + Attributes: + kms_client: The boto3 KMS client + kms_key_id (str): The KMS key ID to use + enable_legacy_wrapping_algorithms (bool): Whether to enable legacy wrapping algorithms + """ kms_client = field() kms_key_id: str = field() enable_legacy_wrapping_algorithms: bool = field(default=False) @@ -54,8 +67,10 @@ def on_decrypt(self, dec_materials, encrypted_data_keys=None): """Decrypt one of the encrypted data keys and update dec_materials. Args: - dec_materials (DecryptionMaterials): A DecryptionMaterials instance containing decryption materials - encrypted_data_keys (List[EncryptedDataKey], optional): A list of encrypted data keys to try. + dec_materials (DecryptionMaterials): A DecryptionMaterials instance containing + decryption materials + encrypted_data_keys (List[EncryptedDataKey], optional): A list of encrypted data + keys to try. Returns: DecryptionMaterials: The updated dec_materials with the plaintext data key (PDK) @@ -86,7 +101,8 @@ def on_decrypt(self, dec_materials, encrypted_data_keys=None): # Default EC MUST NOT be passed in via request if KMS_CONTEXT_DEFAULT_KEY in encryption_context_from_request: raise S3EncryptionClientError( - f"{KMS_CONTEXT_DEFAULT_KEY} is a reserved key for the S3 encryption client" + f"{KMS_CONTEXT_DEFAULT_KEY} is a reserved key for the " + f"S3 encryption client" ) # The stored EC, minus default key/values, MUST match provided EC @@ -96,14 +112,16 @@ def on_decrypt(self, dec_materials, encrypted_data_keys=None): if encryption_context_stored_copy != encryption_context_from_request: # TODO: modeled error raise S3EncryptionClientError( - "Provided encryption context does not match information retrieved from S3" + "Provided encryption context does not match information " + "retrieved from S3" ) # Update decMaterials with the modified encryption context elif edk.key_provider_info == "kms": if not self.enable_legacy_wrapping_algorithms: raise S3EncryptionClientError( - f"Enable legacy wrapping algorithms to use legacy key wrapping algorithm: {edk.key_provider_info}" + f"Enable legacy wrapping algorithms to use legacy key wrapping " + f"algorithm: {edk.key_provider_info}" ) else: raise S3EncryptionClientError( diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index 1ab235da..9a2727f9 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -1,5 +1,11 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +"""Materials module for S3 Encryption Client. + +This module provides classes for encryption and decryption materials, +which contain the cryptographic materials needed for S3 object encryption +and decryption operations. +""" from typing import Any from attrs import define, field diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py index c59370bb..b4378990 100644 --- a/src/s3_encryption/metadata.py +++ b/src/s3_encryption/metadata.py @@ -1,5 +1,10 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +"""Metadata handling for S3 Encryption Client. + +This module provides classes and utilities for managing encryption metadata +for S3 objects, including serialization and deserialization of metadata. +""" import json from typing import Any @@ -16,7 +21,8 @@ class ObjectMetadata: All fields are optional and correspond to the following S3 encryption headers: - encrypted_data_key_v1: The encrypted data key (legacy format) - encrypted_data_key_v2: The encrypted data key (current format) - - encrypted_data_key_algorithm: The algorithm used to encrypt the data key (e.g. AES/GCM or kms+context) + - encrypted_data_key_algorithm: The algorithm used to encrypt the data key + (e.g. AES/GCM or kms+context) - encrypted_data_key_context: The encryption context used for the data key - content_iv: The initialization vector used for content encryption - content_cipher: The cipher algorithm used for content encryption (e.g. AES/GCM/NoPadding) diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 6ce08661..6046fb3a 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -1,5 +1,10 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +"""Encryption and decryption pipelines for S3 Encryption Client. + +This module provides pipelines for encrypting objects before they are put into S3 +and decrypting objects after they are retrieved from S3. +""" import base64 import os @@ -26,7 +31,7 @@ def encrypt(self, plaintext, encryption_context=None): """Encrypt the data before it is stored in S3. Args: - data (bytes or str): The data to be encrypted + plaintext (bytes or str): The data to be encrypted encryption_context (dict, optional): Additional context for encryption Returns: @@ -85,7 +90,7 @@ class GetEncryptedObjectPipeline: cmm: AbstractCryptoMaterialsManager = field() - def decrypt(self, response, encryption_context={}): + def decrypt(self, response, encryption_context=None): """Decrypt the data after it is retrieved from S3. Args: @@ -100,6 +105,10 @@ def decrypt(self, response, encryption_context={}): encryption_metadata = response.get("Metadata", {}) metadata = ObjectMetadata.from_dict(encryption_metadata) + # Use empty dict if encryption_context is None + if encryption_context is None: + encryption_context = {} + iv_b64 = metadata.content_iv edk_b64 = metadata.encrypted_data_key_v2 @@ -141,6 +150,4 @@ def decrypt(self, response, encryption_context={}): aesgcm = AESGCM(dec_materials.plaintext_data_key) - plaintext = aesgcm.decrypt(nonce=iv_bytes, data=encrypted_data, associated_data=None) - - return plaintext + return aesgcm.decrypt(nonce=iv_bytes, data=encrypted_data, associated_data=None) From 6c0b92a6d4679ee6fe90c9e73d3c95e7aefa7ebc Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Tue, 12 Aug 2025 09:53:30 -0700 Subject: [PATCH 31/81] enforce ruff for src --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 47ace280..814ab334 100644 --- a/Makefile +++ b/Makefile @@ -11,8 +11,9 @@ install: # Run linting checks lint: uv run black --check . - # Allow ruff to fail for now as we're gradually adopting linting standards - uv run ruff check src/ test/ || true + # Enforce ruff checks on src/ but allow test/ to fail + uv run ruff check src/ + uv run ruff check test/ || true # Format code with Black and Ruff format: From bee921849d630f3bb8a3de92dd386034ecb583fc Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Tue, 12 Aug 2025 09:54:38 -0700 Subject: [PATCH 32/81] now fix black --- src/s3_encryption/__init__.py | 1 + src/s3_encryption/exceptions.py | 2 ++ src/s3_encryption/materials/crypto_materials_manager.py | 2 ++ src/s3_encryption/materials/kms_keyring.py | 1 + test/test_decryption_materials.py | 1 - test/test_decryption_materials_integration.py | 1 - test/test_encryption_materials.py | 1 - test/test_encryption_materials_integration.py | 1 - test/test_metadata.py | 1 - 9 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 31ccf0a6..bf111d7b 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -33,6 +33,7 @@ class S3EncryptionClient: This client wraps a boto3 S3 client and provides encryption and decryption capabilities for S3 objects using the configured keyring and crypto materials manager. """ + wrapped_s3_client = field() config: S3EncryptionClientConfig = field() diff --git a/src/s3_encryption/exceptions.py b/src/s3_encryption/exceptions.py index d7d44df2..748075fc 100644 --- a/src/s3_encryption/exceptions.py +++ b/src/s3_encryption/exceptions.py @@ -4,5 +4,7 @@ This module contains custom exception classes used throughout the S3 Encryption Client. """ + + class S3EncryptionClientError(Exception): """Exception class for S3 Encryption Client errors.""" diff --git a/src/s3_encryption/materials/crypto_materials_manager.py b/src/s3_encryption/materials/crypto_materials_manager.py index 3ecb2489..349237f3 100644 --- a/src/s3_encryption/materials/crypto_materials_manager.py +++ b/src/s3_encryption/materials/crypto_materials_manager.py @@ -21,6 +21,7 @@ class AbstractCryptoMaterialsManager(abc.ABC): A crypto materials manager is responsible for generating encryption materials and processing decryption materials using a keyring. """ + @abc.abstractmethod def get_encryption_materials(self, enc_mats_request): """Get encryption materials from the keyring. @@ -57,6 +58,7 @@ class DefaultCryptoMaterialsManager(AbstractCryptoMaterialsManager): Attributes: keyring (AbstractKeyring): The keyring to use for cryptographic operations """ + keyring: AbstractKeyring def get_encryption_materials(self, enc_mats_request): diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index 303a5f93..c663d264 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -27,6 +27,7 @@ class KmsKeyring(S3Keyring): kms_key_id (str): The KMS key ID to use enable_legacy_wrapping_algorithms (bool): Whether to enable legacy wrapping algorithms """ + kms_client = field() kms_key_id: str = field() enable_legacy_wrapping_algorithms: bool = field(default=False) diff --git a/test/test_decryption_materials.py b/test/test_decryption_materials.py index 94cd0c4e..aa544cee 100644 --- a/test/test_decryption_materials.py +++ b/test/test_decryption_materials.py @@ -1,7 +1,6 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -import pytest from src.s3_encryption.materials.encrypted_data_key import EncryptedDataKey from src.s3_encryption.materials.materials import DecryptionMaterials diff --git a/test/test_decryption_materials_integration.py b/test/test_decryption_materials_integration.py index 7a20e163..1cfab083 100644 --- a/test/test_decryption_materials_integration.py +++ b/test/test_decryption_materials_integration.py @@ -1,7 +1,6 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -import pytest from unittest.mock import MagicMock, patch from src.s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager diff --git a/test/test_encryption_materials.py b/test/test_encryption_materials.py index bd180d59..9aff14b0 100644 --- a/test/test_encryption_materials.py +++ b/test/test_encryption_materials.py @@ -1,7 +1,6 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -import pytest from src.s3_encryption.materials.encrypted_data_key import EncryptedDataKey from src.s3_encryption.materials.materials import EncryptionMaterials diff --git a/test/test_encryption_materials_integration.py b/test/test_encryption_materials_integration.py index abac6f8c..989d17d8 100644 --- a/test/test_encryption_materials_integration.py +++ b/test/test_encryption_materials_integration.py @@ -1,7 +1,6 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -import pytest from unittest.mock import MagicMock from src.s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager diff --git a/test/test_metadata.py b/test/test_metadata.py index 2abd2866..a061c185 100644 --- a/test/test_metadata.py +++ b/test/test_metadata.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 import os import sys -import pytest # Add the src directory to the Python path sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../src"))) From 4b5ab01a6086666e4d21ba0eabfcb6bef3bc45d2 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Tue, 12 Aug 2025 10:45:34 -0700 Subject: [PATCH 33/81] empty body plus tests --- src/s3_encryption/__init__.py | 5 +- test/integration/test_i_s3_encryption.py | 61 ++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index bf111d7b..62ca99e6 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -45,7 +45,8 @@ def put_object(self, **kwargs): Args: **kwargs: Arguments to pass to the S3 client's put_object method. - Must include Bucket, Key, and Body parameters. + Must include Bucket and Key parameters. + Body parameter is optional; if not provided, an empty object is uploaded. May include EncryptionContext for additional authenticated data. Returns: @@ -54,7 +55,7 @@ def put_object(self, **kwargs): # Extract required parameters from kwargs bucket = kwargs.pop("Bucket") key = kwargs.pop("Key") - body = kwargs.pop("Body") + body = kwargs.pop("Body", b"") # Default to empty bytes when Body is not provided encryption_context = kwargs.pop("EncryptionContext", None) # Create a pipeline for this operation diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index 14692c41..99f25f85 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -40,3 +40,64 @@ def test_simple_roundtrip(): print(output) raise RuntimeError print("Success!") + + +def test_empty_string_roundtrip(): + key = "empty-string-rt" + key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + + data = "" # Empty string as test data + + kms_client = boto3.client("kms", region_name=region) + + keyring = KmsKeyring(kms_client, kms_key_id) + + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + get_req = {"Bucket": bucket, "Key": key} + response = s3ec.get_object(**get_req) + output = response["Body"].read().decode("utf-8") + if output != data: + print("Uh oh! Input and output don't match!") + print("Input:") + print(repr(data)) # Using repr to clearly show it's an empty string + print("Output:") + print(repr(output)) + raise RuntimeError + print("Success! Empty string encrypted and decrypted correctly.") + + +def test_no_body_roundtrip(): + key = "no-body-rt" + key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + + # Expected data when no Body is provided (empty bytes) + expected_data = b"" + + kms_client = boto3.client("kms", region_name=region) + + keyring = KmsKeyring(kms_client, kms_key_id) + + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + # Call put_object without providing a Body parameter + s3ec.put_object(Bucket=bucket, Key=key) + + get_req = {"Bucket": bucket, "Key": key} + response = s3ec.get_object(**get_req) + output = response["Body"].read() + + if output != expected_data: + print("Uh oh! Output doesn't match expected empty bytes!") + print("Expected:") + print(repr(expected_data)) + print("Output:") + print(repr(output)) + raise RuntimeError + print( + "Success! Object with no Body parameter encrypted and decrypted correctly as empty bytes." + ) From c440af4b35255dcf24f63d3795ed7fb9b95c4994 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Tue, 12 Aug 2025 10:51:30 -0700 Subject: [PATCH 34/81] fix type hints --- src/s3_encryption/materials/crypto_materials_manager.py | 3 ++- src/s3_encryption/materials/keyring.py | 2 +- src/s3_encryption/materials/kms_keyring.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/s3_encryption/materials/crypto_materials_manager.py b/src/s3_encryption/materials/crypto_materials_manager.py index 349237f3..82eab454 100644 --- a/src/s3_encryption/materials/crypto_materials_manager.py +++ b/src/s3_encryption/materials/crypto_materials_manager.py @@ -65,7 +65,8 @@ def get_encryption_materials(self, enc_mats_request): """Get encryption materials from the keyring. Args: - enc_mats_request (Dict[str, Any]): Request containing encryption parameters + enc_mats_request (Dict[str, Any] or EncryptionMaterials): Request containing encryption + parameters Returns: EncryptionMaterials: The encryption materials diff --git a/src/s3_encryption/materials/keyring.py b/src/s3_encryption/materials/keyring.py index 6115b02f..7c5b3bf8 100644 --- a/src/s3_encryption/materials/keyring.py +++ b/src/s3_encryption/materials/keyring.py @@ -27,7 +27,7 @@ def on_encrypt(self, enc_materials): """Process encryption materials. Args: - enc_materials (EncryptionMaterials): Encryption materials to process + enc_materials (EncryptionMaterials or dict): Encryption materials to process Returns: EncryptionMaterials: The processed encryption materials diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index c663d264..206b0e92 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -36,7 +36,7 @@ def on_encrypt(self, enc_materials): """Process encryption materials using KMS. Args: - enc_materials (EncryptionMaterials): Encryption materials to process + enc_materials (EncryptionMaterials or dict): Encryption materials to process Returns: EncryptionMaterials: The processed encryption materials with KMS-generated keys From d113d54cc79ff9bc9a1ff365bf14e517f76c4a8d Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Tue, 12 Aug 2025 10:57:45 -0700 Subject: [PATCH 35/81] try client type hint again --- src/s3_encryption/materials/kms_keyring.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index 206b0e92..0715cc8b 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -7,6 +7,7 @@ """ from attrs import define, field +from botocore import client from ..exceptions import S3EncryptionClientError from .encrypted_data_key import EncryptedDataKey @@ -23,12 +24,12 @@ class KmsKeyring(S3Keyring): This keyring uses AWS KMS to generate and decrypt data keys. Attributes: - kms_client: The boto3 KMS client + kms_client (client.BaseClient): The boto3 KMS client kms_key_id (str): The KMS key ID to use enable_legacy_wrapping_algorithms (bool): Whether to enable legacy wrapping algorithms """ - kms_client = field() + kms_client: client.BaseClient = field() kms_key_id: str = field() enable_legacy_wrapping_algorithms: bool = field(default=False) From a73d544ed7b5a68249cf2f5b352ca634e422e816 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Tue, 12 Aug 2025 11:10:17 -0700 Subject: [PATCH 36/81] PDK to plaintext_data_key --- src/s3_encryption/materials/keyring.py | 2 +- src/s3_encryption/materials/kms_keyring.py | 2 +- src/s3_encryption/materials/materials.py | 12 ++++++------ test/test_decryption_materials.py | 4 ++-- test/test_encryption_materials.py | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/s3_encryption/materials/keyring.py b/src/s3_encryption/materials/keyring.py index 7c5b3bf8..1e08cb18 100644 --- a/src/s3_encryption/materials/keyring.py +++ b/src/s3_encryption/materials/keyring.py @@ -45,7 +45,7 @@ def on_decrypt(self, dec_materials, encrypted_data_keys=None): keys to try. Returns: - DecryptionMaterials: The updated dec_materials with the plaintext data key (PDK) + DecryptionMaterials: The updated dec_materials with the plaintext data key """ pass diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index 0715cc8b..7bc8f7bd 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -75,7 +75,7 @@ def on_decrypt(self, dec_materials, encrypted_data_keys=None): keys to try. Returns: - DecryptionMaterials: The updated dec_materials with the plaintext data key (PDK) + DecryptionMaterials: The updated dec_materials with the plaintext data key """ try: # Call parent class validation diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index 9a2727f9..9f72ab91 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -23,7 +23,7 @@ class EncryptionMaterials: Attributes: encryption_context (Dict[str, str]): Context information for encryption encrypted_data_key (Optional[EncryptedDataKey]): The encrypted data key - plaintext_data_key (Optional[bytes]): The plaintext data key (PDK) + plaintext_data_key (Optional[bytes]): The plaintext data key """ encryption_context: dict[str, str] = field(factory=dict) @@ -43,7 +43,7 @@ def from_dict(cls, materials_dict: dict[str, Any]) -> "EncryptionMaterials": return cls( encryption_context=materials_dict.get("encryption_context", {}), encrypted_data_key=materials_dict.get("encrypted_data_key"), - plaintext_data_key=materials_dict.get("PDK"), + plaintext_data_key=materials_dict.get("plaintext_data_key"), ) def to_dict(self) -> dict[str, Any]: @@ -61,7 +61,7 @@ def to_dict(self) -> dict[str, Any]: result["encrypted_data_key"] = self.encrypted_data_key if self.plaintext_data_key is not None: - result["PDK"] = self.plaintext_data_key + result["plaintext_data_key"] = self.plaintext_data_key return result @@ -78,7 +78,7 @@ class DecryptionMaterials: encrypted_data_keys (List[EncryptedDataKey]): List of encrypted data keys to try encryption_context_stored (Dict[str, str]): Encryption context stored with the object encryption_context_from_request (Dict[str, str]): Encryption context provided in the request - plaintext_data_key (Optional[bytes]): The plaintext data key (PDK) + plaintext_data_key (Optional[bytes]): The plaintext data key """ iv: bytes | None = field(default=None) @@ -104,7 +104,7 @@ def from_dict(cls, materials_dict: dict[str, Any]) -> "DecryptionMaterials": encryption_context_from_request=materials_dict.get( "encryption_context_from_request", {} ), - plaintext_data_key=materials_dict.get("PDK"), + plaintext_data_key=materials_dict.get("plaintext_data_key"), ) def to_dict(self) -> dict[str, Any]: @@ -128,6 +128,6 @@ def to_dict(self) -> dict[str, Any]: result["encryption_context_from_request"] = self.encryption_context_from_request if self.plaintext_data_key is not None: - result["PDK"] = self.plaintext_data_key + result["plaintext_data_key"] = self.plaintext_data_key return result diff --git a/test/test_decryption_materials.py b/test/test_decryption_materials.py index aa544cee..c160b509 100644 --- a/test/test_decryption_materials.py +++ b/test/test_decryption_materials.py @@ -56,7 +56,7 @@ def test_from_dict(self): "encrypted_data_keys": [edk], "encryption_context_stored": {"key1": "value1"}, "encryption_context_from_request": {"key2": "value2"}, - "PDK": b"plaintext-data-key", + "plaintext_data_key": b"plaintext-data-key", } materials = DecryptionMaterials.from_dict(materials_dict) assert materials.iv == b"initialization-vector" @@ -84,4 +84,4 @@ def test_to_dict(self): assert materials_dict["encrypted_data_keys"] == [edk] assert materials_dict["encryption_context_stored"] == {"key1": "value1"} assert materials_dict["encryption_context_from_request"] == {"key2": "value2"} - assert materials_dict["PDK"] == b"plaintext-data-key" + assert materials_dict["plaintext_data_key"] == b"plaintext-data-key" diff --git a/test/test_encryption_materials.py b/test/test_encryption_materials.py index 9aff14b0..54d80146 100644 --- a/test/test_encryption_materials.py +++ b/test/test_encryption_materials.py @@ -30,7 +30,7 @@ def test_from_dict(self): materials_dict = { "encryption_context": {"key1": "value1"}, "encrypted_data_key": edk, - "PDK": b"plaintext-data-key", + "plaintext_data_key": b"plaintext-data-key", } materials = EncryptionMaterials.from_dict(materials_dict) assert materials.encryption_context == {"key1": "value1"} @@ -52,4 +52,4 @@ def test_to_dict(self): materials_dict = materials.to_dict() assert materials_dict["encryption_context"] == {"key1": "value1"} assert materials_dict["encrypted_data_key"] == edk - assert materials_dict["PDK"] == b"plaintext-data-key" + assert materials_dict["plaintext_data_key"] == b"plaintext-data-key" From af724e640104e90bb23644d687e273d46a1b4160 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 13 Aug 2025 09:22:25 -0700 Subject: [PATCH 37/81] enforce type checking for put_object, encode to utf-8 to match boto3 --- src/s3_encryption/__init__.py | 44 +++++- test/integration/test_i_s3_encryption.py | 188 ++++++++++++++++++++++- 2 files changed, 226 insertions(+), 6 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 62ca99e6..adfb6886 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -4,8 +4,10 @@ import io from attrs import define, field +from botocore import serialize from botocore.response import StreamingBody +from .exceptions import S3EncryptionClientError from .materials.crypto_materials_manager import ( AbstractCryptoMaterialsManager, DefaultCryptoMaterialsManager, @@ -13,6 +15,8 @@ from .materials.keyring import AbstractKeyring from .pipelines import GetEncryptedObjectPipeline, PutEncryptedObjectPipeline +DEFAULT_ENCODING = "utf-8" + @define class S3EncryptionClientConfig: @@ -37,6 +41,21 @@ class S3EncryptionClient: wrapped_s3_client = field() config: S3EncryptionClientConfig = field() + def __attrs_post_init__(self): + """Validate serialization encoding after initialization. + + Ensures boto3 serializers are using the expected default encoding. + """ + # Sanity check that boto3 serialization are ONLY using the default encoding (utf-8) + # This should always be the case, but changes in encoding would break the assumption that + # the decrypted plaintext adheres to the non-utf8 encoding scheme. So we avoid that. + for sz_name, sz in serialize.SERIALIZERS.items(): + if sz.DEFAULT_ENCODING != DEFAULT_ENCODING: + raise S3EncryptionClientError( + f"All Serializers MUST only support utf-8 encoding, but {sz_name} is using " + f"{sz.DEFAULT_ENCODING}!" + ) + def put_object(self, **kwargs): """Encrypt and upload an object to S3. @@ -61,12 +80,27 @@ def put_object(self, **kwargs): # Create a pipeline for this operation pipeline = PutEncryptedObjectPipeline(self.config.cmm) - # Encrypt the data using the pipeline - data_bytes = body - # We probably just shouldn't support strings, use utf8 for now - # TODO: look deeper into this, what does normal boto3 do? + # The documentation for boto3 asks for bytes or a file-like object, + # but in reality, it is possible to pass strings. + # Strings will be encoded using DEFAULT_ENCODING, + # which MUST match the default encoding defined int the Serializer class in botocore. if isinstance(body, str): - data_bytes = body.encode("utf-8") + data_bytes = body.encode(DEFAULT_ENCODING) + elif isinstance(body, bytes): + data_bytes = body + elif isinstance(body, io.IOBase): + # TODO: Streaming support + raise S3EncryptionClientError( + f"Body parameter of type {type(body)} is not an acceptable type! " + f"Streaming operations are not yet supported." + ) + else: + raise S3EncryptionClientError( + f"Body parameter of type {type(body)} is not an acceptable type! " + f"Use bytes or a file-like object." + ) + + # Now encrypt the bytes/file-like IOBase object encrypted_data, encryption_metadata = pipeline.encrypt( data_bytes, encryption_context=encryption_context ) diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index 99f25f85..2c8ea73a 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -4,8 +4,10 @@ from datetime import datetime import boto3 +import pytest from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.exceptions import S3EncryptionClientError from s3_encryption.materials.kms_keyring import KmsKeyring bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") @@ -15,7 +17,7 @@ ) -def test_simple_roundtrip(): +def test_simple_roundtrip_ascii_string(): key = "simple-rt" key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") @@ -101,3 +103,187 @@ def test_no_body_roundtrip(): print( "Success! Object with no Body parameter encrypted and decrypted correctly as empty bytes." ) + + +def test_unicode_string_roundtrip(): + key = "unicode-string-rt" + key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + + # String with unusual Unicode characters + data = "Unicode test: 你好, こんにちは, 안녕하세요, Привет, مرحبا, ¡Hola!, ½⅓¼⅕⅙⅐⅛⅑⅒⅔⅖⅗⅘⅙⅚⅜⅝⅞" + + kms_client = boto3.client("kms", region_name=region) + + keyring = KmsKeyring(kms_client, kms_key_id) + + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + get_req = {"Bucket": bucket, "Key": key} + response = s3ec.get_object(**get_req) + + # Boto3 encodes to utf-8 in put_object but does not + # decode in get_object; do so manually to complete the + # round trip + output = response["Body"].read().decode("utf-8") + if output != data: + print("Uh oh! Input and output don't match!") + print("Input:") + print(repr(data)) + print("Output:") + print(repr(output)) + raise RuntimeError + print("Success! Unicode string encrypted and decrypted correctly.") + + +def test_specific_encoding_utf8_roundtrip(): + key = "utf8-encoding-rt" + key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + + # String with mixed characters + data = "UTF-8 encoding test: 你好, こんにちは, 안녕하세요, Привет, مرحبا, ¡Hola!" + + # Explicitly encode as UTF-8 before sending + encoded_data = data.encode("utf-8") + + kms_client = boto3.client("kms", region_name=region) + + keyring = KmsKeyring(kms_client, kms_key_id) + + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + # Pass the pre-encoded bytes to put_object + s3ec.put_object(Bucket=bucket, Key=key, Body=encoded_data) + + get_req = {"Bucket": bucket, "Key": key} + response = s3ec.get_object(**get_req) + + # Read raw bytes and decode with the same encoding + output = response["Body"].read().decode("utf-8") + + if output != data: + print("Uh oh! Input and output don't match!") + print("Input:") + print(repr(data)) + print("Output:") + print(repr(output)) + raise RuntimeError + print("Success! UTF-8 encoded string encrypted and decrypted correctly.") + + +def test_specific_encoding_latin1_roundtrip(): + key = "latin1-encoding-rt" + key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + + # String with Latin-1 compatible characters + data = "Latin-1 encoding test: éèêë àâäãåá çñ ¿¡ øæå ØÆÅÉÈÊËÀÂÄÃÅÁ" + + # Explicitly encode as Latin-1 before sending + encoded_data = data.encode("latin-1") + + kms_client = boto3.client("kms", region_name=region) + + keyring = KmsKeyring(kms_client, kms_key_id) + + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + # Pass the pre-encoded bytes to put_object + s3ec.put_object(Bucket=bucket, Key=key, Body=encoded_data) + + get_req = {"Bucket": bucket, "Key": key} + response = s3ec.get_object(**get_req) + + # Read raw bytes and decode with the same encoding + output = response["Body"].read().decode("latin-1") + + if output != data: + print("Uh oh! Input and output don't match!") + print("Input:") + print(repr(data)) + print("Output:") + print(repr(output)) + raise RuntimeError + print("Success! Latin-1 encoded string encrypted and decrypted correctly.") + + +def test_binary_data_roundtrip(): + key = "binary-data-rt" + key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + + # Create some binary data (not valid in any particular encoding) + data = bytes([i for i in range(256)]) + + kms_client = boto3.client("kms", region_name=region) + + keyring = KmsKeyring(kms_client, kms_key_id) + + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + # Pass the binary data directly + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + + get_req = {"Bucket": bucket, "Key": key} + response = s3ec.get_object(**get_req) + + # Read raw bytes without decoding + output = response["Body"].read() + + if output != data: + print("Uh oh! Input and output don't match!") + print("Input:") + print(repr(data)) + print("Output:") + print(repr(output)) + raise RuntimeError + print("Success! Binary data encrypted and decrypted correctly.") + + +def test_invalid_body_types(): + """Test that put_object raises an exception when given invalid body types.""" + key = "invalid-body-type" + key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + # Test with integer + with pytest.raises(S3EncryptionClientError) as excinfo: + s3ec.put_object(Bucket=bucket, Key=key, Body=42) + assert "not an acceptable type" in str(excinfo.value) + + # Test with float + with pytest.raises(S3EncryptionClientError) as excinfo: + s3ec.put_object(Bucket=bucket, Key=key, Body=3.14) + assert "not an acceptable type" in str(excinfo.value) + + # Test with list + with pytest.raises(S3EncryptionClientError) as excinfo: + s3ec.put_object(Bucket=bucket, Key=key, Body=[1, 2, 3]) + assert "not an acceptable type" in str(excinfo.value) + + # Test with dictionary + with pytest.raises(S3EncryptionClientError) as excinfo: + s3ec.put_object(Bucket=bucket, Key=key, Body={"key": "value"}) + assert "not an acceptable type" in str(excinfo.value) + + # Test with boolean + with pytest.raises(S3EncryptionClientError) as excinfo: + s3ec.put_object(Bucket=bucket, Key=key, Body=True) + assert "not an acceptable type" in str(excinfo.value) + + # Test with None (also raises an exception) + with pytest.raises(S3EncryptionClientError) as excinfo: + s3ec.put_object(Bucket=bucket, Key=key, Body=None) + assert "not an acceptable type" in str(excinfo.value) + + print("Success! All invalid body types correctly raised exceptions.") From 8abfabbb6331bf516291829bde070dc0e817f653 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 15 Aug 2025 17:05:03 -0700 Subject: [PATCH 38/81] add test-server and CI for it --- .github/workflows/test.yml | 7 + .gitignore | 39 +- cdk/lib/cdk-stack.ts | 39 +- test-server/Makefile | 99 ++++ test-server/README.md | 44 ++ test-server/java-server/README.md | 23 + test-server/java-server/build.gradle.kts | 55 +++ test-server/java-server/gradle.properties | 3 + .../gradle/wrapper/gradle-wrapper.properties | 7 + test-server/java-server/gradlew | 249 ++++++++++ test-server/java-server/gradlew.bat | 92 ++++ test-server/java-server/license.txt | 4 + test-server/java-server/settings.gradle.kts | 19 + test-server/java-server/smithy-build.json | 11 + .../s3/CreateClientOperationImpl.java | 110 +++++ .../encryption/s3/GetObjectOperationImpl.java | 74 +++ .../amazon/encryption/s3/MetadataUtils.java | 43 ++ .../encryption/s3/PutObjectOperationImpl.java | 58 +++ .../encryption/s3/S3ECJavaTestServer.java | 54 +++ test-server/java-tests/.gitignore | 21 + test-server/java-tests/README.md | 13 + test-server/java-tests/build.gradle.kts | 55 +++ test-server/java-tests/gradle.properties | 3 + .../gradle/wrapper/gradle-wrapper.properties | 7 + test-server/java-tests/gradlew | 249 ++++++++++ test-server/java-tests/gradlew.bat | 92 ++++ test-server/java-tests/license.txt | 4 + test-server/java-tests/settings.gradle.kts | 19 + test-server/java-tests/smithy-build.json | 12 + .../amazon/encryption/s3/RoundTripTests.java | 439 ++++++++++++++++++ test-server/model/client.smithy | 37 ++ test-server/model/main.smithy | 34 ++ test-server/model/object.smithy | 103 ++++ test-server/python-server/.gitignore | 37 ++ test-server/python-server/README.md | 42 ++ test-server/python-server/pyproject.toml | 21 + test-server/python-server/src/__init__.py | 3 + test-server/python-server/src/main.py | 239 ++++++++++ test-server/python-server/tests/__init__.py | 3 + 39 files changed, 2458 insertions(+), 5 deletions(-) create mode 100644 test-server/Makefile create mode 100644 test-server/README.md create mode 100644 test-server/java-server/README.md create mode 100644 test-server/java-server/build.gradle.kts create mode 100644 test-server/java-server/gradle.properties create mode 100644 test-server/java-server/gradle/wrapper/gradle-wrapper.properties create mode 100755 test-server/java-server/gradlew create mode 100644 test-server/java-server/gradlew.bat create mode 100644 test-server/java-server/license.txt create mode 100644 test-server/java-server/settings.gradle.kts create mode 100644 test-server/java-server/smithy-build.json create mode 100644 test-server/java-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java create mode 100644 test-server/java-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java create mode 100644 test-server/java-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java create mode 100644 test-server/java-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java create mode 100644 test-server/java-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java create mode 100644 test-server/java-tests/.gitignore create mode 100644 test-server/java-tests/README.md create mode 100644 test-server/java-tests/build.gradle.kts create mode 100644 test-server/java-tests/gradle.properties create mode 100644 test-server/java-tests/gradle/wrapper/gradle-wrapper.properties create mode 100755 test-server/java-tests/gradlew create mode 100644 test-server/java-tests/gradlew.bat create mode 100644 test-server/java-tests/license.txt create mode 100644 test-server/java-tests/settings.gradle.kts create mode 100644 test-server/java-tests/smithy-build.json create mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java create mode 100644 test-server/model/client.smithy create mode 100644 test-server/model/main.smithy create mode 100644 test-server/model/object.smithy create mode 100644 test-server/python-server/.gitignore create mode 100644 test-server/python-server/README.md create mode 100644 test-server/python-server/pyproject.toml create mode 100644 test-server/python-server/src/__init__.py create mode 100755 test-server/python-server/src/main.py create mode 100644 test-server/python-server/tests/__init__.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0bdc88de..e212267c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,3 +46,10 @@ jobs: env: CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} + + - name: Run test-server tests + run: cd test-server && make ci + env: + AWS_REGION: us-west-2 + TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} + TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} diff --git a/.gitignore b/.gitignore index 22b9a5f7..af14573f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,3 @@ -.idea -.vscode # Exclude all pycache directories and bytecode __pycache__/ *.pyc @@ -14,3 +12,40 @@ build/ # Uv .uv/ uv.lock + +# Gradle +.gradle/ +gradle-app.setting + +# IDE - IntelliJ IDEA +.idea/ +*.iml +*.iws +*.ipr + +# IDE - VS Code +.vscode/ +.settings/ +.project +.classpath + +# Compiled class files +*.class + +# Log files +*.log + +# Package files +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +*.hprof +.kotlin/ + +.DS_Store +smithy-java-core/out diff --git a/cdk/lib/cdk-stack.ts b/cdk/lib/cdk-stack.ts index 88f31473..cdb7c489 100644 --- a/cdk/lib/cdk-stack.ts +++ b/cdk/lib/cdk-stack.ts @@ -22,7 +22,7 @@ export class S3ECPythonGithub extends cdk.Stack { constructor(scope: Construct, id: string, props?: cdk.StackProps) { super(scope, id, props); - // KMS Key - default policy is fine, + // KMS Keys - default policy is fine, // we use IAM to manage key permissions const S3ECGithubKMSKey = new Key( this, @@ -42,8 +42,28 @@ export class S3ECPythonGithub extends cdk.Stack { targetKey: S3ECGithubKMSKey } ) + + // KMS Key for test-server + const S3ECTestServerKMSKey = new Key( + this, + "S3ECTestServerKMSKey", + { + enableKeyRotation: true, + description: "KMS Key for Test Server GitHub Action Workflow", + } + ) - // S3 bucket + // KMS alias for test-server + const S3ECTestServerKMSKeyAlias = new Alias( + this, + "S3ECTestServerKMSKeyAlias", + { + aliasName: "alias/S3EC-Test-Server-Github-KMS-Key", + targetKey: S3ECTestServerKMSKey + } + ) + + // S3 buckets const AccessConfiguration: BlockPublicAccessOptions = { blockPublicAcls: false, blockPublicPolicy: false, @@ -58,6 +78,16 @@ export class S3ECPythonGithub extends cdk.Stack { blockPublicAccess: new BlockPublicAccess(AccessConfiguration) } ) + + // New bucket for test-server + const S3ECTestServerGithubBucket = new Bucket( + this, + "S3ECTestServerGithubBucket", + { + bucketName: "s3ec-test-server-github-bucket", + blockPublicAccess: new BlockPublicAccess(AccessConfiguration) + } + ) // S3 bucket policy const S3ECGithubS3BucketPolicy = new ManagedPolicy( @@ -75,6 +105,7 @@ export class S3ECPythonGithub extends cdk.Stack { ], resources: [ S3ECGithubTestS3Bucket.bucketArn + "/*", // object-level permissions need this extra path + S3ECTestServerGithubBucket.bucketArn + "/*", // Add permissions for the new test-server bucket ], }), new PolicyStatement({ @@ -83,7 +114,8 @@ export class S3ECPythonGithub extends cdk.Stack { "s3:ListBucket", ], resources: [ - S3ECGithubTestS3Bucket.bucketArn + S3ECGithubTestS3Bucket.bucketArn, + S3ECTestServerGithubBucket.bucketArn, // Add permissions for the new test-server bucket ], }), ] @@ -107,6 +139,7 @@ export class S3ECPythonGithub extends cdk.Stack { ], resources: [ S3ECGithubKMSKey.keyArn, + S3ECTestServerKMSKey.keyArn, // Add access to the test-server KMS key ] }) ] diff --git a/test-server/Makefile b/test-server/Makefile new file mode 100644 index 00000000..9ba8a289 --- /dev/null +++ b/test-server/Makefile @@ -0,0 +1,99 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: all start-servers run-tests stop-servers clean ci check-env help + +# Default target +all: start-servers run-tests + +# CI target for GitHub Actions +ci: start-servers run-tests stop-servers + +# Start both servers in background with output to stdout (default for debugging) +start-servers: + @echo "Starting Python server..." + cd python-server && \ + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + poetry install --no-interaction && \ + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + poetry run python src/main.py + @echo "Starting Java server..." + cd java-server && \ + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + ./gradlew run + @echo "Waiting for servers to be ready..." + @for i in $$(seq 1 60); do \ + if nc -z localhost 8080 && nc -z localhost 8081; then \ + echo "Ports are open, waiting for servers to initialize..."; \ + sleep 5; \ + echo "Both servers are ready!"; \ + break; \ + fi; \ + if [ $$i -eq 60 ]; then \ + echo "Timeout waiting for servers to start"; \ + exit 1; \ + fi; \ + echo "Waiting for servers to start ($$i/60)..."; \ + sleep 1; \ + done + + +# Run the Java tests +run-tests: + @echo "Running Java tests..." + @echo "Exporting environment variables from servers to tests..." + @# Extract AWS environment variables from the current shell and pass them to the tests + cd java-tests && \ + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + ./gradlew integ + @echo "Tests completed successfully" + +# Stop the servers +stop-servers: + @echo "Stopping servers..." + @if [ -f python-server.pid ]; then \ + kill $$(cat python-server.pid) 2>/dev/null || true; \ + rm python-server.pid; \ + fi + @if [ -f java-server.pid ]; then \ + kill $$(cat java-server.pid) 2>/dev/null || true; \ + rm java-server.pid; \ + fi + @echo "Servers stopped" + +# Clean up logs and pid files +clean: stop-servers + @echo "Cleaning up..." + @rm -f python-server.log java-server.log + @echo "Cleanup complete" + +# Help target +help: + @echo "Available targets:" + @echo " all : Start servers and run tests (default, output to stdout)" + @echo " ci : Run in CI mode (start servers, run tests, stop servers)" + @echo " start-servers: Start Python and Java servers (output to stdout)" + @echo " run-tests : Run Java tests" + @echo " stop-servers : Stop running servers" + @echo " clean : Stop servers and clean up logs" + @echo " check-env : Check if required environment variables are set" + @echo " help : Show this help message" + +# Check if required environment variables are set +check-env: + @echo "Checking required environment variables..." + @if [ -z "$$AWS_ACCESS_KEY_ID" ]; then echo "AWS_ACCESS_KEY_ID is not set"; else echo "AWS_ACCESS_KEY_ID is set"; fi + @if [ -z "$$AWS_SECRET_ACCESS_KEY" ]; then echo "AWS_SECRET_ACCESS_KEY is not set"; else echo "AWS_SECRET_ACCESS_KEY is set"; fi + @if [ -z "$$AWS_SESSION_TOKEN" ]; then echo "AWS_SESSION_TOKEN is not set"; else echo "AWS_SESSION_TOKEN is set"; fi + @if [ -z "$$AWS_REGION" ]; then echo "AWS_REGION is not set (will use us-west-2 as default)"; else echo "AWS_REGION is set to $$AWS_REGION"; fi diff --git a/test-server/README.md b/test-server/README.md new file mode 100644 index 00000000..06ce12ed --- /dev/null +++ b/test-server/README.md @@ -0,0 +1,44 @@ +# S3EC Generalized Robust Test Framework Machine + +Or G-RTFM. Or something. + +## What? + +This is an attempt at writing a write-once, run-multiple test server. + +## How? + +Use Smithy Java roughly as it is intended. +That is, generate a client and a server which share a common model. +Then, write more servers, either using the server codegen or parsing the JSON blobject by "hand". + +## Running Tests + +A Makefile is provided to simplify running the servers and tests. The Makefile handles starting both the Python and Java servers, running the tests, and cleaning up. + +### Available Commands + +```bash +# Start servers and run tests (default) +make + +# Run in CI mode (start servers, run tests, stop servers) +make ci + +# Start Python and Java servers +make start-servers + +# Run Java tests +make run-tests + +# Stop running servers +make stop-servers + +# Stop servers and clean up logs +make clean + +# Show help message +make help +``` + +The `ci` target is specifically designed for GitHub Actions workflows, ensuring that servers are properly started, tests are run, and resources are cleaned up afterward. diff --git a/test-server/java-server/README.md b/test-server/java-server/README.md new file mode 100644 index 00000000..b2f5bb1b --- /dev/null +++ b/test-server/java-server/README.md @@ -0,0 +1,23 @@ +# S3EC Java Test Server + +This is the Java implementation of the S3ECTestServer framework. It provides a server implementation for testing S3 Encryption Client functionality. + +## Overview + +The S3ECJavaTestServer implements the S3ECTestServer service defined in the shared Smithy model. It provides endpoints for: + +- Creating S3 Encryption Clients +- Putting objects with encryption +- Getting and decrypting objects + +## Usage + +To run the server: + +```console +gradle run +``` + +This will start the server running on port `8080`. + +The server is used as part of the testing framework to verify cross-language compatibility of the S3 Encryption Client implementations. diff --git a/test-server/java-server/build.gradle.kts b/test-server/java-server/build.gradle.kts new file mode 100644 index 00000000..ca793e56 --- /dev/null +++ b/test-server/java-server/build.gradle.kts @@ -0,0 +1,55 @@ +plugins { + `java-library` + id("software.amazon.smithy.gradle.smithy-base") + application +} + +dependencies { + val smithyJavaVersion: String by project + + smithyBuild("software.amazon.smithy.java:plugins:$smithyJavaVersion") + + implementation("software.amazon.smithy:smithy-rules-engine:1.59.0") + implementation("software.amazon.smithy.java:server-netty:$smithyJavaVersion") + implementation("software.amazon.smithy.java:aws-server-restjson:$smithyJavaVersion") + + compileOnly("software.amazon.awssdk:aws-sdk-java:2.31.66") + implementation("software.amazon.encryption.s3:amazon-s3-encryption-client-java:3.3.5") +} + +// Use that application plugin to start the service via the `run` task. +application { + mainClass = "software.amazon.encryption.s3.S3ECJavaTestServer" +} + +// Add generated Java files to the main sourceSet +afterEvaluate { + val serverPath = smithy.getPluginProjectionPath(smithy.sourceProjection.get(), "java-server-codegen") + sourceSets { + main { + java { + srcDir(serverPath) + } + } + } +} + +tasks { + compileJava { + dependsOn(smithyBuild) + } +} + +// Helps Intellij IDE's discover smithy models +sourceSets { + main { + java { + srcDir("../model") + } + } +} + +repositories { + mavenLocal() + mavenCentral() +} diff --git a/test-server/java-server/gradle.properties b/test-server/java-server/gradle.properties new file mode 100644 index 00000000..0af8556d --- /dev/null +++ b/test-server/java-server/gradle.properties @@ -0,0 +1,3 @@ +smithyJavaVersion=[0,1] +smithyGradleVersion=1.1.0 +smithyVersion=[1,2] diff --git a/test-server/java-server/gradle/wrapper/gradle-wrapper.properties b/test-server/java-server/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..a4413138 --- /dev/null +++ b/test-server/java-server/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/test-server/java-server/gradlew b/test-server/java-server/gradlew new file mode 100755 index 00000000..b740cf13 --- /dev/null +++ b/test-server/java-server/gradlew @@ -0,0 +1,249 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/test-server/java-server/gradlew.bat b/test-server/java-server/gradlew.bat new file mode 100644 index 00000000..7101f8e4 --- /dev/null +++ b/test-server/java-server/gradlew.bat @@ -0,0 +1,92 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/test-server/java-server/license.txt b/test-server/java-server/license.txt new file mode 100644 index 00000000..2dd564b3 --- /dev/null +++ b/test-server/java-server/license.txt @@ -0,0 +1,4 @@ +/* + * Example file license header. + * File header line two + */ \ No newline at end of file diff --git a/test-server/java-server/settings.gradle.kts b/test-server/java-server/settings.gradle.kts new file mode 100644 index 00000000..c608c023 --- /dev/null +++ b/test-server/java-server/settings.gradle.kts @@ -0,0 +1,19 @@ +/** + * Basic usage of generated server stubs. + */ + +pluginManagement { + val smithyGradleVersion: String by settings + + plugins { + id("software.amazon.smithy.gradle.smithy-base").version(smithyGradleVersion) + } + + repositories { + mavenLocal() + mavenCentral() + gradlePluginPortal() + } +} + +rootProject.name = "BasicSmithyJavaServer" diff --git a/test-server/java-server/smithy-build.json b/test-server/java-server/smithy-build.json new file mode 100644 index 00000000..a0fcb8e5 --- /dev/null +++ b/test-server/java-server/smithy-build.json @@ -0,0 +1,11 @@ +{ + "version": "2.0", + "plugins": { + "java-server-codegen": { + "service": "software.amazon.encryption.s3#S3ECTestServer", + "namespace": "software.amazon.encryption.s3", + "headerFile": "license.txt" + } + }, + "sources": ["../model"] +} diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java b/test-server/java-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java new file mode 100644 index 00000000..f798eb5d --- /dev/null +++ b/test-server/java-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java @@ -0,0 +1,110 @@ +package software.amazon.encryption.s3; + +import software.amazon.awssdk.core.traits.Trait; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.encryption.s3.S3EncryptionClient; +import software.amazon.encryption.s3.materials.AesKeyring; +import software.amazon.encryption.s3.materials.Keyring; +import software.amazon.encryption.s3.materials.KmsKeyring; +import software.amazon.encryption.s3.materials.PartialRsaKeyPair; +import software.amazon.encryption.s3.materials.RsaKeyring; +import software.amazon.smithy.java.core.schema.Schema; +import software.amazon.smithy.java.server.RequestContext; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.GenericServerError; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.service.CreateClientOperation; + +import javax.crypto.spec.SecretKeySpec; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.security.KeyFactory; +import java.security.NoSuchAlgorithmException; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + +public class CreateClientOperationImpl implements CreateClientOperation { + private Map clientCache_; + + public CreateClientOperationImpl(Map clientCache) { + clientCache_ = clientCache; + } + + // Copied from S3EC. + private boolean onlyOneNonNull(Object... values) { + boolean haveOneNonNull = false; + for (Object o : values) { + if (o != null) { + if (haveOneNonNull) { + return false; + } + + haveOneNonNull = true; + } + } + + return haveOneNonNull; + } + + @Override + public CreateClientOutput createClient(CreateClientInput input, RequestContext context) { + System.out.println("createClient called!"); + try { + KeyMaterial key = input.config().keyMaterial(); + if (!onlyOneNonNull(key.aesKey(), key.kmsKeyId(), key.rsaKey())) { + throw new RuntimeException("KeyMaterial must be only one, non-null input!"); + } + Keyring keyring; + if (key.aesKey() != null) { + byte[] keyBytes = new byte[key.aesKey().remaining()]; + key.aesKey().get(keyBytes); + keyring = AesKeyring.builder() + .wrappingKey(new SecretKeySpec(keyBytes, "AES")) + .enableLegacyWrappingAlgorithms(input.config().enableLegacyWrappingAlgorithms()) + .build(); + } else if (key.rsaKey() != null) { + try { + byte[] keyBytes = new byte[key.rsaKey().remaining()]; + key.rsaKey().get(keyBytes); + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(keyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + keyring = RsaKeyring.builder() + .enableLegacyWrappingAlgorithms(input.config().enableLegacyWrappingAlgorithms()) + .wrappingKeyPair(PartialRsaKeyPair.builder() + .privateKey(keyFactory.generatePrivate(keySpec)).build()) + .build(); + } catch (NoSuchAlgorithmException | InvalidKeySpecException nse) { + throw new RuntimeException(nse); + } + } else if (key.kmsKeyId() != null) { + keyring = KmsKeyring.builder() + .enableLegacyWrappingAlgorithms(input.config().enableLegacyWrappingAlgorithms()) + .wrappingKeyId(key.kmsKeyId()) + .build(); + } else { + throw new RuntimeException("No KeyMaterial found!"); + } + S3Client s3Client = S3EncryptionClient.builder() + .keyring(keyring) + .build(); + UUID uuid = UUID.randomUUID(); + String uuidString = uuid.toString(); + clientCache_.put(uuidString, s3Client); + return CreateClientOutput.builder() + .clientId(uuidString) + .build(); + } catch (Exception e) { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw GenericServerError.builder() + .message(stackTrace) + .build(); + } + } +} diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java b/test-server/java-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java new file mode 100644 index 00000000..95529511 --- /dev/null +++ b/test-server/java-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java @@ -0,0 +1,74 @@ +package software.amazon.encryption.s3; + +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.encryption.s3.S3EncryptionClientException; +import software.amazon.smithy.java.server.RequestContext; +import software.amazon.encryption.s3.model.GenericServerError; +import software.amazon.encryption.s3.model.GetObjectInput; +import software.amazon.encryption.s3.model.GetObjectOutput; +import software.amazon.encryption.s3.model.S3EncryptionClientError; +import software.amazon.encryption.s3.service.GetObjectOperation; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static software.amazon.encryption.s3.S3EncryptionClient.withAdditionalConfiguration; +import static software.amazon.encryption.s3.MetadataUtils.metadataListToMap; +import static software.amazon.encryption.s3.MetadataUtils.metadataMapToList; + +public class GetObjectOperationImpl implements GetObjectOperation { + private Map clientCache_; + public GetObjectOperationImpl(Map clientCache) { + clientCache_ = clientCache; + } + @Override + public GetObjectOutput getObject(GetObjectInput input, RequestContext context) { + try { + System.out.println("Getting object with ClientId: " + input.clientID()); + S3Client s3Client = clientCache_.get(input.clientID()); + Map ecMap = metadataListToMap(input.metadata()); + + try { + ResponseBytes resp = s3Client.getObjectAsBytes(builder -> builder + .bucket(input.bucket()) + .key(input.key()) + .overrideConfiguration(withAdditionalConfiguration(ecMap))); + + List mdAsList = metadataMapToList(resp.response().metadata()); + // Can't use asBB else it gets mad bc cant access backing array + ByteBuffer bb = ByteBuffer.wrap(resp.asByteArray()); + GetObjectOutput output = GetObjectOutput.builder() + .body(bb) + .metadata(mdAsList) + .build(); + System.out.println("returning"); + return output; + } catch (S3EncryptionClientException s3EncryptionClientException) { + // Modeled exceptions MUST be returned as such + StringWriter sw = new StringWriter(); + s3EncryptionClientException.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw S3EncryptionClientError.builder() + .message(stackTrace) + .build(); + } + } catch (Exception e) { + // Don't wrap modeled errors + if (e instanceof S3EncryptionClientError) { + throw e; + } + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw GenericServerError.builder() + .message(stackTrace) + .build(); + } + } +} diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java b/test-server/java-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java new file mode 100644 index 00000000..036289ec --- /dev/null +++ b/test-server/java-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java @@ -0,0 +1,43 @@ +package software.amazon.encryption.s3; + +import software.amazon.encryption.s3.model.GenericServerError; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class MetadataUtils { + + /** + * Annoyingly, Smithy doesn't provide an interface for map types + * in HTTP headers, so we have to do the serde ourselves + */ + public static List metadataMapToList(Map md) { + List mdAsList = new ArrayList<>(md.size()); + for (Map.Entry keyValue : md.entrySet()) { + mdAsList.add("[" + keyValue.getKey() + "]:[" + keyValue.getValue() + "]"); + } + return mdAsList; + } + + public static Map metadataListToMap(List mdList) { + Map md = new HashMap<>(); + for (String entry : mdList) { + // Split on "]:[" to separate key and value + String[] parts = entry.split("]:\\["); + if (parts.length == 2) { + // Remove remaining brackets from start and end + String key = parts[0].substring(1); + String value = parts[1].substring(0, parts[1].length() - 1); + md.put(key, value); + } else { + throw GenericServerError.builder() + .message("Malformed metadata list entry: " + entry) + .build(); + } + } + return md; + } + +} diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java b/test-server/java-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java new file mode 100644 index 00000000..c7bbbbe2 --- /dev/null +++ b/test-server/java-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java @@ -0,0 +1,58 @@ +package software.amazon.encryption.s3; + +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.smithy.java.server.RequestContext; +import software.amazon.encryption.s3.model.GenericServerError; +import software.amazon.encryption.s3.model.PutObjectInput; +import software.amazon.encryption.s3.model.PutObjectOutput; +import software.amazon.encryption.s3.service.PutObjectOperation; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Map; +import java.util.stream.Collectors; + +import static software.amazon.encryption.s3.S3EncryptionClient.withAdditionalConfiguration; +import static software.amazon.encryption.s3.MetadataUtils.metadataListToMap; + +public class PutObjectOperationImpl implements PutObjectOperation { + + private Map clientCache_; + + public PutObjectOperationImpl(Map clientCache) { + clientCache_ = clientCache; + } + + @Override + public PutObjectOutput putObject(PutObjectInput input, RequestContext context) { + try { + System.out.println("Putting object with ClientId: " + input.clientID()); + System.out.println("putting " + input.key() + " in bucket " + input.bucket() + " with content: " + input.body()); + final Map metadata = metadataListToMap(input.metadata()); + S3Client s3Client = clientCache_.get(input.clientID()); + s3Client.putObject(builder -> builder + .bucket(input.bucket()) + .key(input.key()) + .overrideConfiguration(withAdditionalConfiguration(metadata)), + RequestBody.fromByteBuffer(input.body()) + ); + System.out.println("Success!"); + // The real S3 doesn't provide bucket/key/metadata, so Test doesn't need to either, but we do anyway + return PutObjectOutput.builder() + .bucket(input.bucket()) + .key(input.key()) + .metadata(input.metadata()) + .build(); + } catch (Exception e) { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw GenericServerError.builder() + .message(stackTrace) + .build(); + } + } +} diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java b/test-server/java-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java new file mode 100644 index 00000000..327966c6 --- /dev/null +++ b/test-server/java-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java @@ -0,0 +1,54 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.encryption.s3; + +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.encryption.s3.S3EncryptionClient; + +import java.net.URI; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import software.amazon.smithy.java.server.Server; +import software.amazon.encryption.s3.service.S3ECTestServer; + +public class S3ECJavaTestServer implements Runnable { + static final URI endpoint = URI.create("http://localhost:8080"); + + public static void main(String[] args) { + new S3ECJavaTestServer().run(); + } + + @Override + public void run() { + // All the S3EC instances live here. + // Obviously this can get messy in a real service. + // Assume that the tests behave and don't induce weird race conditions. + Map clientCache = new ConcurrentHashMap<>(); + + Server server = Server.builder() + .endpoints(endpoint) + .addService( + S3ECTestServer.builder() + .addCreateClientOperation(new CreateClientOperationImpl(clientCache)) + .addGetObjectOperation(new GetObjectOperationImpl(clientCache)) + .addPutObjectOperation(new PutObjectOperationImpl(clientCache)) + .build()) + .build(); + System.out.println("Starting server..."); + server.start(); + try { + Thread.currentThread().join(); + } catch (InterruptedException e) { + System.out.println("Stopping server..."); + try { + server.shutdown().get(); + } catch (InterruptedException | ExecutionException ex) { + throw new RuntimeException(ex); + } + } + } +} diff --git a/test-server/java-tests/.gitignore b/test-server/java-tests/.gitignore new file mode 100644 index 00000000..0cde3479 --- /dev/null +++ b/test-server/java-tests/.gitignore @@ -0,0 +1,21 @@ +# Ignore Gradle project-specific cache directory +.gradle + +# Ignore kotlin cache dir +.kotlin + +# Ignore Gradle build output directory +build + +# Ignore intellij files +.idea + +#Ignore mac files +.DS_Store + +# Intellij stuff +.classpath +.project +.settings + +smithy-java-core/out diff --git a/test-server/java-tests/README.md b/test-server/java-tests/README.md new file mode 100644 index 00000000..eee84863 --- /dev/null +++ b/test-server/java-tests/README.md @@ -0,0 +1,13 @@ +## Java Tests + +This project contains Java client tests for the S3 Encryption Client. + +### Running Tests + +To run the integration tests for this project: + +```console +gradle integ +``` + +The integration tests will connect to the appropriate test servers automatically. diff --git a/test-server/java-tests/build.gradle.kts b/test-server/java-tests/build.gradle.kts new file mode 100644 index 00000000..f35a2ac6 --- /dev/null +++ b/test-server/java-tests/build.gradle.kts @@ -0,0 +1,55 @@ +plugins { + `java-library` + id("software.amazon.smithy.gradle.smithy-base") +} + +dependencies { + val smithyJavaVersion: String by project + + smithyBuild("software.amazon.smithy.java:plugins:$smithyJavaVersion") + implementation("software.amazon.smithy:smithy-rules-engine:1.59.0") + + // Client dependencies + implementation("software.amazon.smithy.java:aws-client-restjson:$smithyJavaVersion") + implementation("software.amazon.smithy.java:client-core:$smithyJavaVersion") + + // Test dependencies + testImplementation("org.junit.jupiter:junit-jupiter:5.13.0") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") + testImplementation("com.amazonaws:aws-java-sdk:1.12.788") + testImplementation("org.bouncycastle:bcprov-jdk15on:1.70") +} + +// Add generated Java sources to the main sourceset +afterEvaluate { + val clientPath = smithy.getPluginProjectionPath(smithy.sourceProjection.get(), "java-client-codegen") + sourceSets { + main { + java { + srcDir(clientPath) + } + } + create("it") { + compileClasspath += main.get().output + configurations["testRuntimeClasspath"] + configurations["testCompileClasspath"] + runtimeClasspath += output + compileClasspath + test.get().runtimeClasspath + test.get().output + } + } +} + +tasks { + val smithyBuild by getting + compileJava { + dependsOn(smithyBuild) + } + + val integ by registering(Test::class) { + useJUnitPlatform() + testClassesDirs = sourceSets["it"].output.classesDirs + classpath = sourceSets["it"].runtimeClasspath + } +} + +repositories { + mavenLocal() + mavenCentral() +} diff --git a/test-server/java-tests/gradle.properties b/test-server/java-tests/gradle.properties new file mode 100644 index 00000000..0af8556d --- /dev/null +++ b/test-server/java-tests/gradle.properties @@ -0,0 +1,3 @@ +smithyJavaVersion=[0,1] +smithyGradleVersion=1.1.0 +smithyVersion=[1,2] diff --git a/test-server/java-tests/gradle/wrapper/gradle-wrapper.properties b/test-server/java-tests/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..a4413138 --- /dev/null +++ b/test-server/java-tests/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/test-server/java-tests/gradlew b/test-server/java-tests/gradlew new file mode 100755 index 00000000..b740cf13 --- /dev/null +++ b/test-server/java-tests/gradlew @@ -0,0 +1,249 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/test-server/java-tests/gradlew.bat b/test-server/java-tests/gradlew.bat new file mode 100644 index 00000000..7101f8e4 --- /dev/null +++ b/test-server/java-tests/gradlew.bat @@ -0,0 +1,92 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/test-server/java-tests/license.txt b/test-server/java-tests/license.txt new file mode 100644 index 00000000..edaafd85 --- /dev/null +++ b/test-server/java-tests/license.txt @@ -0,0 +1,4 @@ +/* + * Example file license header. + * File header line two + */ diff --git a/test-server/java-tests/settings.gradle.kts b/test-server/java-tests/settings.gradle.kts new file mode 100644 index 00000000..ae20971f --- /dev/null +++ b/test-server/java-tests/settings.gradle.kts @@ -0,0 +1,19 @@ +/** + * Java client tests for S3 Encryption Client. + */ + +pluginManagement { + val smithyGradleVersion: String by settings + + plugins { + id("software.amazon.smithy.gradle.smithy-base").version(smithyGradleVersion) + } + + repositories { + mavenLocal() + mavenCentral() + gradlePluginPortal() + } +} + +rootProject.name = "Java-Tests" diff --git a/test-server/java-tests/smithy-build.json b/test-server/java-tests/smithy-build.json new file mode 100644 index 00000000..3fe72762 --- /dev/null +++ b/test-server/java-tests/smithy-build.json @@ -0,0 +1,12 @@ +{ + "version": "1.0", + "plugins": { + "java-client-codegen": { + "service": "software.amazon.encryption.s3#S3ECTestServer", + "namespace": "software.amazon.encryption.s3", + "headerFile": "license.txt", + "protocol": "aws.protocols#restJson1" + } + }, + "sources": ["../model"] +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java new file mode 100644 index 00000000..b7851e11 --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -0,0 +1,439 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.encryption.s3; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.net.Socket; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Stream; + +import com.amazonaws.services.s3.model.KMSEncryptionMaterials; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.smithy.java.aws.client.restjson.RestJsonClientProtocol; +import software.amazon.smithy.java.client.core.ClientConfig; +import software.amazon.smithy.java.client.core.ClientProtocol; +import software.amazon.smithy.java.client.core.endpoint.EndpointResolver; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.GetObjectInput; +import software.amazon.encryption.s3.model.GetObjectOutput; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.PutObjectInput; +import software.amazon.encryption.s3.model.S3ECConfig; +import software.amazon.encryption.s3.model.S3ECTestServerApiService; +import software.amazon.encryption.s3.model.S3EncryptionClientError; +import software.amazon.smithy.java.http.api.HttpRequest; +import software.amazon.smithy.java.http.api.HttpResponse; + +import com.amazonaws.regions.Region; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3Encryption; +import com.amazonaws.services.s3.AmazonS3EncryptionClient; +import com.amazonaws.services.s3.model.CryptoConfiguration; +import com.amazonaws.services.s3.model.CryptoMode; +import com.amazonaws.services.s3.model.CryptoStorageMode; +import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; + +public class RoundTripTests { + private static final List serverList; + private static final Map serverMap; + + private static final String KMS_KEY_ARN = System.getenv("TEST_SERVER_KMS_KEY_ARN") != null ? + System.getenv("TEST_SERVER_KMS_KEY_ARN") : "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Test-Server-Github-KMS-Key"; + private static final Region KMS_REGION = Region.getRegion(Regions.fromName("us-west-2")); + private static final String BUCKET = System.getenv("TEST_SERVER_S3_BUCKET") != null ? + System.getenv("TEST_SERVER_S3_BUCKET") : "s3ec-test-server-github-bucket"; + + static { + serverList = new ArrayList<>(2); + serverList.add(new LanguageServerTarget("Java", "8080")); + serverList.add(new LanguageServerTarget("Python", "8081")); + + serverMap = new HashMap<>(2); + serverMap.put("Java", new LanguageServerTarget("Java", "8080")); + serverMap.put("Python", new LanguageServerTarget("Python", "8081")); + } + + static public class LanguageServerTarget { + public String getLangaugeName() { + return langaugeName; + } + + public URI getServerURI() { + return serverURI; + } + + private final String baseURI = "http://localhost"; + private String langaugeName; + private URI serverURI; + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + LanguageServerTarget that = (LanguageServerTarget) o; + return Objects.equals(langaugeName, that.langaugeName) && Objects.equals(serverURI, that.serverURI); + } + + @Override + public int hashCode() { + return Objects.hash(langaugeName, serverURI); + } + + LanguageServerTarget(String language, String port) { + langaugeName = language; + serverURI = URI.create(baseURI+ ":" + port); + } + + @Override + public String toString() { + return langaugeName; + } + } + + @BeforeAll + public static void setup() { + // Wait for servers to start + for (LanguageServerTarget server : serverList) { + if (!serverListening(server.getServerURI())) { + throw new RuntimeException(String.format("Test Server for %s is not running at endpoint: %s", server.getLangaugeName(), server.getServerURI())); + } + } + } + + public static boolean serverListening(URI uri) { + try (Socket ignored = new Socket(uri.getHost(), uri.getPort())) { + return true; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + + static S3ECTestServerClient testServerClientFor(LanguageServerTarget server) { + S3ECTestServerApiService apiService = S3ECTestServerApiService.instance(); + ClientProtocol rest = new RestJsonClientProtocol(apiService.schema().id()); + return S3ECTestServerClient.builder() + .endpointResolver(EndpointResolver.staticEndpoint(server.serverURI)) + .withConfiguration(ClientConfig.builder() + .service(apiService) + .protocol(rest) + .endpointResolver(EndpointResolver.staticEndpoint(server.serverURI)) + .build()) + .build(); + } + + static Stream clientsForTest() { + return serverList.stream() + .map(LanguageServerTarget::getLangaugeName) + .map(Arguments::of); + } + + static Stream crossLanguageClients() { + return serverList.stream() + .flatMap(t1 -> serverList.stream() +// .filter(t2 -> !t1.equals(t2)) + .flatMap(t2 -> Stream.of( + Arguments.of(t1, t2) + ))); + } + + /** + * Annoyingly, Smithy doesn't provide an interface for map types + * in HTTP headers, so we have to do the serde ourselves + * Servers need an equivalent utility. + * TODO: Move to a utilities class or something. + */ + private List metadataMapToList(Map md) { + List mdAsList = new ArrayList<>(md.size()); + for (Map.Entry keyValue : md.entrySet()) { + // Using ":" because Smithy will parse "," into a flattened list + mdAsList.add("[" + keyValue.getKey() + "]:[" + keyValue.getValue() + "]"); + } + return mdAsList; + } + + @ParameterizedTest(name = "{displayName} for Encrypt: {0}, Decrypt: {1}") + @MethodSource("crossLanguageClients") + public void crossLanguageTestKms(LanguageServerTarget encLang, LanguageServerTarget decLang) { + S3ECTestServerClient encClient = testServerClientFor(encLang); + final String objectKey = "cross-lang-test-key-" + encLang; + final String input = "simple-test-input"; + KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(KMS_KEY_ARN) + .build(); + CreateClientOutput encClientOutput = encClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn).build()) + .build()); + String encS3ECId = encClientOutput.clientId(); + encClient.putObject(PutObjectInput.builder() + .clientID(encS3ECId) + .key(objectKey) + .bucket(BUCKET) + .body(ByteBuffer.wrap(input.getBytes(StandardCharsets.UTF_8))) + .build()); + S3ECTestServerClient decClient = testServerClientFor(decLang); + CreateClientOutput decClientOutput = decClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn).build()) + .build()); + String decS3ECId = decClientOutput.clientId(); + GetObjectOutput output = decClient.getObject(GetObjectInput.builder() + .clientID(decS3ECId) + .bucket(BUCKET) + .key(objectKey) + .build()); + + if (!input.equals(StandardCharsets.UTF_8.decode(output.body()).toString())) { + System.out.println(String.format("Encryption in %s failed to decrpyt in %s!", encLang, decLang)); + fail(String.format("Encryption in %s failed to decrpyt in %s!", encLang, decLang)); + } + } + + @ParameterizedTest(name = "{displayName} for Encrypt: {0}, Decrypt: {1}") + @MethodSource("crossLanguageClients") + public void crossLanguageTestKmsWithEncCtx(LanguageServerTarget encLang, LanguageServerTarget decLang) { + S3ECTestServerClient encClient = testServerClientFor(encLang); + final String objectKey = "cross-lang-test-key-kms-ec-" + encLang; + final String input = "simple-test-input"; + final Map encCtx = new HashMap<>(); + encCtx.put("user-defined-enc-ctx-key", "user-defined-enc-ctx-value"); + encCtx.put("user-defined-enc-ctx-key-2", "user-defined-enc-ctx-value-2"); + final List mdAsList = metadataMapToList(encCtx); + KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(KMS_KEY_ARN) + .build(); + CreateClientOutput encClientOutput = encClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn).build()) + .build()); + String encS3ECId = encClientOutput.clientId(); + + encClient.putObject(PutObjectInput.builder() + .clientID(encS3ECId) + .key(objectKey) + .bucket(BUCKET) + .metadata(mdAsList) + .body(ByteBuffer.wrap(input.getBytes(StandardCharsets.UTF_8))) + .build()); + S3ECTestServerClient decClient = testServerClientFor(decLang); + CreateClientOutput decClientOutput = decClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn).build()) + .build()); + String decS3ECId = decClientOutput.clientId(); + GetObjectOutput output = decClient.getObject(GetObjectInput.builder() + .clientID(decS3ECId) + .bucket(BUCKET) + .key(objectKey) + .metadata(mdAsList) + .build()); + + if (!input.equals(StandardCharsets.UTF_8.decode(output.body()).toString())) { + System.out.println(String.format("Encryption in %s failed to decrpyt in %s!", encLang, decLang)); + fail(String.format("Encryption in %s failed to decrpyt in %s!", encLang, decLang)); + } + } + + @ParameterizedTest(name = "{displayName} for Encrypt: {0}, Decrypt: {1}") + @MethodSource("crossLanguageClients") + public void crossLanguageTestKmsWithEncCtxMismatchFails(LanguageServerTarget encLang, LanguageServerTarget decLang) { + S3ECTestServerClient encClient = testServerClientFor(encLang); + final String objectKey = "cross-lang-test-key-kms-ec-mismatch-fails" + encLang; + final String input = "simple-test-input"; + final Map encCtx = new HashMap<>(); + encCtx.put("user-defined-enc-ctx-key", "user-defined-enc-ctx-value"); + encCtx.put("user-defined-enc-ctx-key-2", "user-defined-enc-ctx-value-2"); + final List mdAsList = metadataMapToList(encCtx); + KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(KMS_KEY_ARN) + .build(); + CreateClientOutput encClientOutput = encClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn).build()) + .build()); + String encS3ECId = encClientOutput.clientId(); + + encClient.putObject(PutObjectInput.builder() + .clientID(encS3ECId) + .key(objectKey) + .bucket(BUCKET) + .metadata(mdAsList) + .body(ByteBuffer.wrap(input.getBytes(StandardCharsets.UTF_8))) + .build()); + S3ECTestServerClient decClient = testServerClientFor(decLang); + CreateClientOutput decClientOutput = decClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn).build()) + .build()); + String decS3ECId = decClientOutput.clientId(); + try { + decClient.getObject(GetObjectInput.builder() + .clientID(decS3ECId) + .bucket(BUCKET) + .key(objectKey) + .build()); + fail("Expected exception!"); + } catch (S3EncryptionClientError e) { + assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3")); + } + } + + @ParameterizedTest(name = "{displayName} for Encrypt: Java, Decrypt: {0}") + @MethodSource("clientsForTest") + public void kmsV1Legacy(String language) { + S3ECTestServerClient client = testServerClientFor(serverMap.get(language)); + final String objectKey = "test-key-kms-v1-" + language; + final String input = "simple-test-input"; + KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(KMS_KEY_ARN) + .build(); + CreateClientOutput output1 = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .enableLegacyWrappingAlgorithms(true) + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String s3ECId = output1.clientId(); + + // Create the object using the old client + // V1 Client + EncryptionMaterialsProvider materialsProvider = new KMSEncryptionMaterialsProvider(KMS_KEY_ARN); + + CryptoConfiguration v1Config = + new CryptoConfiguration(CryptoMode.AuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.ObjectMetadata) + .withAwsKmsRegion(KMS_REGION); + + AmazonS3Encryption v1Client = AmazonS3EncryptionClient.encryptionBuilder() + .withCryptoConfiguration(v1Config) + .withEncryptionMaterials(materialsProvider) + .build(); + + v1Client.putObject(BUCKET, objectKey, input); + + GetObjectOutput output = client.getObject(GetObjectInput.builder() + .clientID(s3ECId) + .bucket(BUCKET) + .key(objectKey) + .build()); + + assertEquals(input, new String(output.body().array())); + } + + @ParameterizedTest(name = "{displayName} for Encrypt: Java, Decrypt: {0}") + @MethodSource("clientsForTest") + public void kmsV1LegacyWithEncCtx(String language) { + S3ECTestServerClient client = testServerClientFor(serverMap.get(language)); + final String objectKey = "test-key-kms-v1-with-enc-ctx-" + language; + final String input = "simple-test-input"; + KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(KMS_KEY_ARN) + .build(); + CreateClientOutput output1 = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .enableLegacyWrappingAlgorithms(true) + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String s3ECId = output1.clientId(); + + // Create the object using the old client + // V1 Client + final String ecKey = "user-metadata-key"; + final String ecValue = "user-metadata-value-v1"; + KMSEncryptionMaterials kmsMaterials = new KMSEncryptionMaterials(KMS_KEY_ARN); + kmsMaterials.addDescription(ecKey, ecValue); + EncryptionMaterialsProvider materialsProvider = new KMSEncryptionMaterialsProvider(kmsMaterials); + + CryptoConfiguration v1Config = + new CryptoConfiguration(CryptoMode.AuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.ObjectMetadata) + .withAwsKmsRegion(KMS_REGION); + + AmazonS3Encryption v1Client = AmazonS3EncryptionClient.encryptionBuilder() + .withCryptoConfiguration(v1Config) + .withEncryptionMaterials(materialsProvider) + .build(); + + v1Client.putObject(BUCKET, objectKey, input); + + final Map encCtx = new HashMap<>(); + encCtx.put(ecKey, ecValue); + GetObjectOutput output = client.getObject(GetObjectInput.builder() + .clientID(s3ECId) + .bucket(BUCKET) + .key(objectKey) + .metadata(metadataMapToList(encCtx)) + .build()); + + assertEquals(input, new String(output.body().array())); + } + + @ParameterizedTest(name = "{displayName} for Encrypt: Java, Decrypt: {0}") + @MethodSource("clientsForTest") + public void kmsV1LegacyFailsWhenLegacyDisabled(String language) { + S3ECTestServerClient client = testServerClientFor(serverMap.get(language)); + final String objectKey = "test-key-kms-v1-fails-disabled" + language; + final String input = "simple-test-input"; + KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(KMS_KEY_ARN) + .build(); + CreateClientOutput output1 = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .enableLegacyWrappingAlgorithms(false) + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String s3ECId = output1.clientId(); + + // Create the object using the old client + // V1 Client + EncryptionMaterialsProvider materialsProvider = new KMSEncryptionMaterialsProvider(KMS_KEY_ARN); + + CryptoConfiguration v1Config = + new CryptoConfiguration(CryptoMode.AuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.ObjectMetadata) + .withAwsKmsRegion(KMS_REGION); + + AmazonS3Encryption v1Client = AmazonS3EncryptionClient.encryptionBuilder() + .withCryptoConfiguration(v1Config) + .withEncryptionMaterials(materialsProvider) + .build(); + + v1Client.putObject(BUCKET, objectKey, input); + + try { + client.getObject(GetObjectInput.builder() + .clientID(s3ECId) + .bucket(BUCKET) + .key(objectKey) + .build()); + fail("Expected Exception"); + } catch (S3EncryptionClientError e) { + assertTrue(e.getMessage().contains("Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms")); + } + } + +} diff --git a/test-server/model/client.smithy b/test-server/model/client.smithy new file mode 100644 index 00000000..4de56b5b --- /dev/null +++ b/test-server/model/client.smithy @@ -0,0 +1,37 @@ +$version: "2.0" + +namespace software.amazon.encryption.s3 + +/// Client Creation/Configuration +@http(method: "POST", uri: "/client") +operation CreateClient { + input: CreateClientInput, + output: CreateClientOutput, +} + +@input +structure CreateClientInput { + config: S3ECConfig, +} + +@output +structure CreateClientOutput { + clientId: String, +} + +/// Since it's possible to pass this directly, include it separately +/// Probably also need a Keyring structure to signal when to create Keyrings directly +/// Or maybe KeyringConfig +structure KeyMaterial { + rsaKey: Blob, + aesKey: Blob, + kmsKeyId: String +} + +structure S3ECConfig { + enableLegacyUnauthenticatedModes: Boolean = false, + enableDelayedAuthenticationMode: Boolean = false, + enableLegacyWrappingAlgorithms: Boolean = false, + setBufferSize: Long, + keyMaterial: KeyMaterial +} diff --git a/test-server/model/main.smithy b/test-server/model/main.smithy new file mode 100644 index 00000000..0f7611b5 --- /dev/null +++ b/test-server/model/main.smithy @@ -0,0 +1,34 @@ +$version: "2" + +namespace software.amazon.encryption.s3 + +use aws.protocols#restJson1 + +@title("S3 Encryption Client Test Service") +@restJson1 +service S3ECTestServer { + version: "2024-08-23" + operations: [ + CreateClient + ] + resources: [ + Object + ] + errors: [GenericServerError, S3EncryptionClientError] +} + +/// Used for "internal" errors, e.g. problems with the test server itself +/// Tests MUST NOT expect this error in negative tests. +@error("server") +structure GenericServerError { + @required + message: String +} + +/// Used for modeled errors, e.g. errors thrown by the S3EC +/// Tests SHOULD expect this error in negative tests. +@error("server") +structure S3EncryptionClientError { + @required + message: String +} diff --git a/test-server/model/object.smithy b/test-server/model/object.smithy new file mode 100644 index 00000000..623d8ed3 --- /dev/null +++ b/test-server/model/object.smithy @@ -0,0 +1,103 @@ +$version: "2.0" + +namespace software.amazon.encryption.s3 + +/// Represents an S3-like bucket +///resource Bucket { +/// identifiers: { +/// bucketName: String +/// } +///} + +/// Represents an S3-like object +resource Object { + identifiers: { + bucket: String + key: String + } + properties: { + body: StreamingBlob + metadata: ObjectMetadata + } + read: GetObject + put: PutObject +} + +@idempotent +@http(method: "PUT", uri: "/object/{bucket}/{key}") +operation PutObject { + input := for Object { + @httpLabel + @required + $bucket + + @httpLabel + @required + $key + + @httpHeader("Content-Metadata") + $metadata + + @required + @httpPayload + $body + + @httpHeader("ClientID") + @required + @notProperty + clientID: String + } + + output := for Object { + @required + $bucket + + @required + $key + + @required + $metadata + } +} + +@readonly +@http(method: "GET", uri: "/object/{bucket}/{key}") +operation GetObject { + input := for Object { + @httpLabel + @required + $bucket + + @httpLabel + @required + $key + + /// Should probably be renamed to be EC specific + @httpHeader("Content-Metadata") + $metadata + + @httpHeader("ClientID") + @required + @notProperty + clientID: String + } + + output := for Object { + @httpHeader("Content-Metadata") + @required + $metadata + + @required + @httpPayload + $body + } +} + +/// Smithy does not know how to serialize a map +list ObjectMetadata { + member: String +} + +/// Seems like Streaming is broken in Java. +///@streaming +blob StreamingBlob diff --git a/test-server/python-server/.gitignore b/test-server/python-server/.gitignore new file mode 100644 index 00000000..1089c7d2 --- /dev/null +++ b/test-server/python-server/.gitignore @@ -0,0 +1,37 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Poetry +poetry.lock +.venv/ +venv/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo + +# Testing +.coverage +htmlcov/ +.pytest_cache/ diff --git a/test-server/python-server/README.md b/test-server/python-server/README.md new file mode 100644 index 00000000..619e030a --- /dev/null +++ b/test-server/python-server/README.md @@ -0,0 +1,42 @@ +# Python Server + +A FastAPI-based Python server implementation. + +## Setup + +1. Install Poetry (if not already installed): +```bash +curl -sSL https://install.python-poetry.org | python3 - +``` + +2. Install dependencies: +```bash +poetry install +``` + +## Development + +- Source code is in the `src` directory +- Tests are in the `tests` directory +- Use `poetry shell` to activate the virtual environment +- Use `poetry add {package}` to add new dependencies +- Use `poetry add -D {package}` to add new development dependencies + +## Running the Server + +```bash +poetry run python src/main.py +``` + +The server will start on `http://localhost:8080` with the following endpoints: +- `GET /` - Welcome message +- `POST /get-beer` - Get a beer with specified ID + - Request body: `{"Id": "string"}` + - Response: `{"beer": "beer{Id}"}` +- `GET /docs` - Interactive API documentation (provided by Swagger UI) +- `GET /redoc` - Alternative API documentation (provided by ReDoc) + +## Running Tests + +```bash +poetry run pytest diff --git a/test-server/python-server/pyproject.toml b/test-server/python-server/pyproject.toml new file mode 100644 index 00000000..5cbe63e6 --- /dev/null +++ b/test-server/python-server/pyproject.toml @@ -0,0 +1,21 @@ +[tool.poetry] +name = "python-server" +version = "0.1.0" +description = "A Python server implementation" +authors = ["Your Name"] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.11" +boto3 = "^1.37.2" +pytest = ">=8.4.1,<9.0.0" +fastapi = "^0.115.12" +uvicorn = "^0.34.2" +amazon-s3-encryption-client-python = { path = "../..", develop = true } + +[tool.poetry.group.dev.dependencies] +pytest-cov = "^6.1.1" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/test-server/python-server/src/__init__.py b/test-server/python-server/src/__init__.py new file mode 100644 index 00000000..84d55777 --- /dev/null +++ b/test-server/python-server/src/__init__.py @@ -0,0 +1,3 @@ +""" +Python server package initialization. +""" diff --git a/test-server/python-server/src/main.py b/test-server/python-server/src/main.py new file mode 100755 index 00000000..a7ccbdea --- /dev/null +++ b/test-server/python-server/src/main.py @@ -0,0 +1,239 @@ +""" +Main entry point for the Python server. +""" +from fastapi import FastAPI, Request, HTTPException, Response, status +from fastapi.responses import JSONResponse +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.materials.kms_keyring import KmsKeyring +import boto3 +import uvicorn +import json +import uuid + +app = FastAPI(title="Python Server") + +# Dictionary to store clients with their UUIDs as keys +client_cache = {} + +# Java gets a list, but since there's no Smithy Python Server, +# this is just a string. +def metadata_string_to_map(md_string): + md = {} + if md_string == '': + return md + md_list = md_string.split(",") + for entry in md_list: + # Split on "]:[" to separate key and value + parts = entry.split("]:[") + if len(parts) == 2: + # Remove remaining brackets from start and end + key = parts[0][1:] # Remove first character + value = parts[1][:-1] # Remove last character + md[key] = value + else: + raise ValueError(f"Malformed metadata list entry: {entry}") + return md + + +def create_generic_server_error(message: str, status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR): + """ + Create a response that matches the GenericServerError type from the Smithy model. + Used for internal server errors. + """ + return JSONResponse( + status_code=status_code, + content={ + "__type": "software.amazon.encryption.s3#GenericServerError", + "message": message + } + ) + +def create_s3_encryption_client_error(message: str, status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR): + """ + Create a response that matches the S3EncryptionClientError type from the Smithy model. + Used for errors thrown by the S3 Encryption Client. + """ + return JSONResponse( + status_code=status_code, + content={ + "__type": "software.amazon.encryption.s3#S3EncryptionClientError", + "message": message + } + ) + +@app.put("/object/{bucket}/{key}") +async def put_object(bucket: str, key: str, request: Request): + """ + Handle PUT requests to /object/{bucket}/{key} by using the S3EncryptionClient + to make a PutObject request to S3. + """ + client_id = request.headers.get("ClientID") + body = await request.body() + print(f"PUT object request - Bucket: {bucket}, Key: {key}") + print(f"ClientID from header: {client_id}") + + if not client_id: + return create_generic_server_error("ClientID header is required", status.HTTP_400_BAD_REQUEST) + + # Get the S3EncryptionClient from the client_cache + client = client_cache.get(client_id) + if not client: + return create_generic_server_error(f"No client found for ClientID: {client_id}", status.HTTP_404_NOT_FOUND) + + try: + metadata = request.headers.get("Content-Metadata", '') + enc_ctx = metadata_string_to_map(metadata) + + # Make the PutObject request + response = client.put_object( + **{ + "Bucket": bucket, + "Key": key, + "Body": body, + "EncryptionContext": enc_ctx + } + ) + + print(f"PutObject response: {response}") + + # Return the appropriate response + return { + "bucket": bucket, + "key": key, + "metadata": metadata if isinstance(metadata, list) else [] + } + except Exception as e: + print(f"Error making PutObject request: {e}") + return create_s3_encryption_client_error(f"Failed to put object: {str(e)}") + +@app.get("/object/{bucket}/{key}") +async def get_object(bucket: str, key: str, request: Request): + """ + Handle GET requests to /object/{bucket}/{key} by using the S3EncryptionClient + to make a GetObject request to S3. + """ + client_id = request.headers.get("ClientID") + print(f"GET object request - Bucket: {bucket}, Key: {key}") + print(f"ClientID from header: {client_id}") + + if not client_id: + return create_generic_server_error("ClientID header is required", status.HTTP_400_BAD_REQUEST) + + # Get the S3EncryptionClient from the client_cache + client = client_cache.get(client_id) + if not client: + return create_generic_server_error(f"No client found for ClientID: {client_id}", status.HTTP_404_NOT_FOUND) + + metadata = request.headers.get("Content-Metadata", '') + enc_ctx = metadata_string_to_map(metadata) + + try: + # Use the client to make a GetObject request to S3 + print("making Get for " + key) + response = client.get_object( + **{ + "Bucket": bucket, + "Key": key, + "EncryptionContext": enc_ctx + } + ) + + print(f"GetObject response: {response}") + + # Extract the body and metadata from the response + body = response.get('Body').read() if response.get('Body') else b'' + # print(f"body:" + body) + metadata = response.get('Metadata', []) + print(f"md: {metadata}") + + # Convert metadata dictionary to a list of key-value pairs if it's a dict + if isinstance(metadata, dict): + metadata_list = [f"{key}={value}" for key, value in metadata.items()] + else: + metadata_list = metadata if isinstance(metadata, list) else [] + + # Set the Content-Metadata header in the response + # Convert metadata_list to a comma-separated string + metadata_str = ",".join(metadata_list) if metadata_list else "" + headers = {"Content-Metadata": metadata_str} + print(f"headers: {headers}") + + # Return the body as the response payload + return Response( + content=body, + headers=headers + ) + except S3EncryptionClientError as ex: + print(f"Modeled Error making GetObject request: {ex}") + return create_s3_encryption_client_error(str(ex)) + except Exception as e: + print(f"Generic Error making GetObject request: {e}") + return create_generic_server_error(e) + +@app.post("/client") +async def client_endpoint(request: Request): + """ + Handle POST requests to /client by creating an S3EncryptionClient. + """ + body = await request.body() + print(f"Received client request with body: {body}") + + # Parse the bytes object as JSON + try: + # Decode bytes to string and parse as JSON + parsed_data = json.loads(body.decode('utf-8')) + print(f"Parsed JSON data: {parsed_data}") + + # Extract config from the parsed data + config_data = parsed_data.get("config", {}) + # Extract key material if provided + key_material = config_data.get("keyMaterial", {}) + if key_material: + # Note: This is a placeholder. The actual implementation would depend on how + # the S3EncryptionClient handles key material + print(f"Key material provided: {key_material}") + + enable_legacy_wrapping_algorithms = config_data.get("enableLegacyWrappingAlgorithms", False) + + # TODO pull region from ARN + kms_client = boto3.client("kms", region_name="us-west-2") + kms_key_id = key_material['kmsKeyId'] + keyring = KmsKeyring(kms_client, kms_key_id=kms_key_id, enable_legacy_wrapping_algorithms=enable_legacy_wrapping_algorithms) + wrapped_client = boto3.client("s3") + client_config = S3EncryptionClientConfig(keyring) + # Create S3EncryptionClientConfig + # client_config = S3EncryptionClientConfig( + # enable_legacy_unauthenticated_modes=config_data.get("enableLegacyUnauthenticatedModes", False), + # enable_delayed_authentication_mode=config_data.get("enableDelayedAuthenticationMode", False), + # enable_legacy_wrapping_algorithms=config_data.get("enableLegacyWrappingAlgorithms", False), + # buffer_size=config_data.get("setBufferSize", 0) + # ) + + # Create S3EncryptionClient + client = S3EncryptionClient(wrapped_client, client_config) + print(f"Created S3EncryptionClient: {client}") + + # Generate a client ID using UUID + client_id = str(uuid.uuid4()) + + # Add the client to the client_cache dictionary + client_cache[client_id] = client + print(f"Added client to cache with ID: {client_id}") + + return {"clientId": client_id} + except json.JSONDecodeError as e: + print(f"Error parsing JSON: {e}") + return create_generic_server_error("Invalid JSON in request body", status.HTTP_400_BAD_REQUEST) + except Exception as e: + print(f"Error creating S3EncryptionClient: {e}") + return create_s3_encryption_client_error(f"Failed to create client: {str(e)}") + +def main(): + """ + Main function to start the server. + """ + uvicorn.run(app, host="localhost", port=8081) + +if __name__ == "__main__": + main() diff --git a/test-server/python-server/tests/__init__.py b/test-server/python-server/tests/__init__.py new file mode 100644 index 00000000..8b28a306 --- /dev/null +++ b/test-server/python-server/tests/__init__.py @@ -0,0 +1,3 @@ +""" +Test package initialization. +""" From 03ef0cbc3f43829a8eb56dde7ad2fb7424d5d119 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 15 Aug 2025 17:08:03 -0700 Subject: [PATCH 39/81] install poetry --- .github/workflows/test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e212267c..670e55da 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,6 +29,13 @@ jobs: - name: Install Uv run: pip install uv + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: latest + virtualenvs-create: true + virtualenvs-in-project: true + - name: Install dependencies run: make install From af32de46b4243796fd37ffbbec6503e70f1b78c3 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 15 Aug 2025 17:10:53 -0700 Subject: [PATCH 40/81] dont package test server --- test-server/python-server/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/test-server/python-server/pyproject.toml b/test-server/python-server/pyproject.toml index 5cbe63e6..a4d2e4ab 100644 --- a/test-server/python-server/pyproject.toml +++ b/test-server/python-server/pyproject.toml @@ -4,6 +4,7 @@ version = "0.1.0" description = "A Python server implementation" authors = ["Your Name"] readme = "README.md" +package-mode = false [tool.poetry.dependencies] python = "^3.11" From c3489d1c3ab5018a5758bbf0e7647b716192fd09 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 15 Aug 2025 17:14:39 -0700 Subject: [PATCH 41/81] fix Makefile --- test-server/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-server/Makefile b/test-server/Makefile index 9ba8a289..4941f3d1 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -21,14 +21,14 @@ start-servers: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - poetry run python src/main.py + poetry run python src/main.py & echo $$! > ../python-server.pid @echo "Starting Java server..." cd java-server && \ AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./gradlew run + ./gradlew run & echo $$! > ../java-server.pid @echo "Waiting for servers to be ready..." @for i in $$(seq 1 60); do \ if nc -z localhost 8080 && nc -z localhost 8081; then \ From 65b0ed4fa47d82ddcc0e5bb0ea26db4f5e256d8e Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 15 Aug 2025 17:19:13 -0700 Subject: [PATCH 42/81] Fix: Include gradle-wrapper.jar files in repository to fix CI build --- .gitignore | 2 ++ .../gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 43453 bytes .../gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 43453 bytes 3 files changed, 2 insertions(+) create mode 100644 test-server/java-server/gradle/wrapper/gradle-wrapper.jar create mode 100644 test-server/java-tests/gradle/wrapper/gradle-wrapper.jar diff --git a/.gitignore b/.gitignore index af14573f..0e29a9fb 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,8 @@ gradle-app.setting # Package files *.jar +!gradle/wrapper/gradle-wrapper.jar +!**/gradle/wrapper/gradle-wrapper.jar *.war *.nar *.ear diff --git a/test-server/java-server/gradle/wrapper/gradle-wrapper.jar b/test-server/java-server/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..e6441136f3d4ba8a0da8d277868979cfbc8ad796 GIT binary patch literal 43453 zcma&N1CXTcmMvW9vTb(Rwr$&4wr$(C?dmSu>@vG-+vuvg^_??!{yS%8zW-#zn-LkA z5&1^$^{lnmUON?}LBF8_K|(?T0Ra(xUH{($5eN!MR#ZihR#HxkUPe+_R8Cn`RRs(P z_^*#_XlXmGv7!4;*Y%p4nw?{bNp@UZHv1?Um8r6)Fei3p@ClJn0ECfg1hkeuUU@Or zDaPa;U3fE=3L}DooL;8f;P0ipPt0Z~9P0)lbStMS)ag54=uL9ia-Lm3nh|@(Y?B`; zx_#arJIpXH!U{fbCbI^17}6Ri*H<>OLR%c|^mh8+)*h~K8Z!9)DPf zR2h?lbDZQ`p9P;&DQ4F0sur@TMa!Y}S8irn(%d-gi0*WxxCSk*A?3lGh=gcYN?FGl z7D=Js!i~0=u3rox^eO3i@$0=n{K1lPNU zwmfjRVmLOCRfe=seV&P*1Iq=^i`502keY8Uy-WNPwVNNtJFx?IwAyRPZo2Wo1+S(xF37LJZ~%i)kpFQ3Fw=mXfd@>%+)RpYQLnr}B~~zoof(JVm^^&f zxKV^+3D3$A1G;qh4gPVjhrC8e(VYUHv#dy^)(RoUFM?o%W-EHxufuWf(l*@-l+7vt z=l`qmR56K~F|v<^Pd*p~1_y^P0P^aPC##d8+HqX4IR1gu+7w#~TBFphJxF)T$2WEa zxa?H&6=Qe7d(#tha?_1uQys2KtHQ{)Qco)qwGjrdNL7thd^G5i8Os)CHqc>iOidS} z%nFEDdm=GXBw=yXe1W-ShHHFb?Cc70+$W~z_+}nAoHFYI1MV1wZegw*0y^tC*s%3h zhD3tN8b=Gv&rj}!SUM6|ajSPp*58KR7MPpI{oAJCtY~JECm)*m_x>AZEu>DFgUcby z1Qaw8lU4jZpQ_$;*7RME+gq1KySGG#Wql>aL~k9tLrSO()LWn*q&YxHEuzmwd1?aAtI zBJ>P=&$=l1efe1CDU;`Fd+_;&wI07?V0aAIgc(!{a z0Jg6Y=inXc3^n!U0Atk`iCFIQooHqcWhO(qrieUOW8X(x?(RD}iYDLMjSwffH2~tB z)oDgNBLB^AJBM1M^c5HdRx6fBfka`(LD-qrlh5jqH~);#nw|iyp)()xVYak3;Ybik z0j`(+69aK*B>)e_p%=wu8XC&9e{AO4c~O1U`5X9}?0mrd*m$_EUek{R?DNSh(=br# z#Q61gBzEpmy`$pA*6!87 zSDD+=@fTY7<4A?GLqpA?Pb2z$pbCc4B4zL{BeZ?F-8`s$?>*lXXtn*NC61>|*w7J* z$?!iB{6R-0=KFmyp1nnEmLsA-H0a6l+1uaH^g%c(p{iT&YFrbQ$&PRb8Up#X3@Zsk zD^^&LK~111%cqlP%!_gFNa^dTYT?rhkGl}5=fL{a`UViaXWI$k-UcHJwmaH1s=S$4 z%4)PdWJX;hh5UoK?6aWoyLxX&NhNRqKam7tcOkLh{%j3K^4Mgx1@i|Pi&}<^5>hs5 zm8?uOS>%)NzT(%PjVPGa?X%`N2TQCKbeH2l;cTnHiHppPSJ<7y-yEIiC!P*ikl&!B z%+?>VttCOQM@ShFguHVjxX^?mHX^hSaO_;pnyh^v9EumqSZTi+#f&_Vaija0Q-e*| z7ulQj6Fs*bbmsWp{`auM04gGwsYYdNNZcg|ph0OgD>7O}Asn7^Z=eI>`$2*v78;sj-}oMoEj&@)9+ycEOo92xSyY344^ z11Hb8^kdOvbf^GNAK++bYioknrpdN>+u8R?JxG=!2Kd9r=YWCOJYXYuM0cOq^FhEd zBg2puKy__7VT3-r*dG4c62Wgxi52EMCQ`bKgf*#*ou(D4-ZN$+mg&7$u!! z-^+Z%;-3IDwqZ|K=ah85OLwkO zKxNBh+4QHh)u9D?MFtpbl)us}9+V!D%w9jfAMYEb>%$A;u)rrI zuBudh;5PN}_6J_}l55P3l_)&RMlH{m!)ai-i$g)&*M`eN$XQMw{v^r@-125^RRCF0 z^2>|DxhQw(mtNEI2Kj(;KblC7x=JlK$@78`O~>V!`|1Lm-^JR$-5pUANAnb(5}B}JGjBsliK4& zk6y(;$e&h)lh2)L=bvZKbvh@>vLlreBdH8No2>$#%_Wp1U0N7Ank!6$dFSi#xzh|( zRi{Uw%-4W!{IXZ)fWx@XX6;&(m_F%c6~X8hx=BN1&q}*( zoaNjWabE{oUPb!Bt$eyd#$5j9rItB-h*5JiNi(v^e|XKAj*8(k<5-2$&ZBR5fF|JA z9&m4fbzNQnAU}r8ab>fFV%J0z5awe#UZ|bz?Ur)U9bCIKWEzi2%A+5CLqh?}K4JHi z4vtM;+uPsVz{Lfr;78W78gC;z*yTch~4YkLr&m-7%-xc ztw6Mh2d>_iO*$Rd8(-Cr1_V8EO1f*^@wRoSozS) zy1UoC@pruAaC8Z_7~_w4Q6n*&B0AjOmMWa;sIav&gu z|J5&|{=a@vR!~k-OjKEgPFCzcJ>#A1uL&7xTDn;{XBdeM}V=l3B8fE1--DHjSaxoSjNKEM9|U9#m2<3>n{Iuo`r3UZp;>GkT2YBNAh|b z^jTq-hJp(ebZh#Lk8hVBP%qXwv-@vbvoREX$TqRGTgEi$%_F9tZES@z8Bx}$#5eeG zk^UsLBH{bc2VBW)*EdS({yw=?qmevwi?BL6*=12k9zM5gJv1>y#ML4!)iiPzVaH9% zgSImetD@dam~e>{LvVh!phhzpW+iFvWpGT#CVE5TQ40n%F|p(sP5mXxna+Ev7PDwA zamaV4m*^~*xV+&p;W749xhb_X=$|LD;FHuB&JL5?*Y2-oIT(wYY2;73<^#46S~Gx| z^cez%V7x$81}UWqS13Gz80379Rj;6~WdiXWOSsdmzY39L;Hg3MH43o*y8ibNBBH`(av4|u;YPq%{R;IuYow<+GEsf@R?=@tT@!}?#>zIIn0CoyV!hq3mw zHj>OOjfJM3F{RG#6ujzo?y32m^tgSXf@v=J$ELdJ+=5j|=F-~hP$G&}tDZsZE?5rX ztGj`!S>)CFmdkccxM9eGIcGnS2AfK#gXwj%esuIBNJQP1WV~b~+D7PJTmWGTSDrR` zEAu4B8l>NPuhsk5a`rReSya2nfV1EK01+G!x8aBdTs3Io$u5!6n6KX%uv@DxAp3F@{4UYg4SWJtQ-W~0MDb|j-$lwVn znAm*Pl!?Ps&3wO=R115RWKb*JKoexo*)uhhHBncEDMSVa_PyA>k{Zm2(wMQ(5NM3# z)jkza|GoWEQo4^s*wE(gHz?Xsg4`}HUAcs42cM1-qq_=+=!Gk^y710j=66(cSWqUe zklbm8+zB_syQv5A2rj!Vbw8;|$@C!vfNmNV!yJIWDQ>{+2x zKjuFX`~~HKG~^6h5FntRpnnHt=D&rq0>IJ9#F0eM)Y-)GpRjiN7gkA8wvnG#K=q{q z9dBn8_~wm4J<3J_vl|9H{7q6u2A!cW{bp#r*-f{gOV^e=8S{nc1DxMHFwuM$;aVI^ zz6A*}m8N-&x8;aunp1w7_vtB*pa+OYBw=TMc6QK=mbA-|Cf* zvyh8D4LRJImooUaSb7t*fVfih<97Gf@VE0|z>NcBwBQze);Rh!k3K_sfunToZY;f2 z^HmC4KjHRVg+eKYj;PRN^|E0>Gj_zagfRbrki68I^#~6-HaHg3BUW%+clM1xQEdPYt_g<2K+z!$>*$9nQ>; zf9Bei{?zY^-e{q_*|W#2rJG`2fy@{%6u0i_VEWTq$*(ZN37|8lFFFt)nCG({r!q#9 z5VK_kkSJ3?zOH)OezMT{!YkCuSSn!K#-Rhl$uUM(bq*jY? zi1xbMVthJ`E>d>(f3)~fozjg^@eheMF6<)I`oeJYx4*+M&%c9VArn(OM-wp%M<-`x z7sLP1&3^%Nld9Dhm@$3f2}87!quhI@nwd@3~fZl_3LYW-B?Ia>ui`ELg z&Qfe!7m6ze=mZ`Ia9$z|ARSw|IdMpooY4YiPN8K z4B(ts3p%2i(Td=tgEHX z0UQ_>URBtG+-?0E;E7Ld^dyZ;jjw0}XZ(}-QzC6+NN=40oDb2^v!L1g9xRvE#@IBR zO!b-2N7wVfLV;mhEaXQ9XAU+>=XVA6f&T4Z-@AX!leJ8obP^P^wP0aICND?~w&NykJ#54x3_@r7IDMdRNy4Hh;h*!u(Ol(#0bJdwEo$5437-UBjQ+j=Ic>Q2z` zJNDf0yO6@mr6y1#n3)s(W|$iE_i8r@Gd@!DWDqZ7J&~gAm1#~maIGJ1sls^gxL9LLG_NhU!pTGty!TbhzQnu)I*S^54U6Yu%ZeCg`R>Q zhBv$n5j0v%O_j{QYWG!R9W?5_b&67KB$t}&e2LdMvd(PxN6Ir!H4>PNlerpBL>Zvyy!yw z-SOo8caEpDt(}|gKPBd$qND5#a5nju^O>V&;f890?yEOfkSG^HQVmEbM3Ugzu+UtH zC(INPDdraBN?P%kE;*Ae%Wto&sgw(crfZ#Qy(<4nk;S|hD3j{IQRI6Yq|f^basLY; z-HB&Je%Gg}Jt@={_C{L$!RM;$$|iD6vu#3w?v?*;&()uB|I-XqEKqZPS!reW9JkLewLb!70T7n`i!gNtb1%vN- zySZj{8-1>6E%H&=V}LM#xmt`J3XQoaD|@XygXjdZ1+P77-=;=eYpoEQ01B@L*a(uW zrZeZz?HJsw_4g0vhUgkg@VF8<-X$B8pOqCuWAl28uB|@r`19DTUQQsb^pfqB6QtiT z*`_UZ`fT}vtUY#%sq2{rchyfu*pCg;uec2$-$N_xgjZcoumE5vSI{+s@iLWoz^Mf; zuI8kDP{!XY6OP~q5}%1&L}CtfH^N<3o4L@J@zg1-mt{9L`s^z$Vgb|mr{@WiwAqKg zp#t-lhrU>F8o0s1q_9y`gQNf~Vb!F%70f}$>i7o4ho$`uciNf=xgJ>&!gSt0g;M>*x4-`U)ysFW&Vs^Vk6m%?iuWU+o&m(2Jm26Y(3%TL; zA7T)BP{WS!&xmxNw%J=$MPfn(9*^*TV;$JwRy8Zl*yUZi8jWYF>==j~&S|Xinsb%c z2?B+kpet*muEW7@AzjBA^wAJBY8i|#C{WtO_or&Nj2{=6JTTX05}|H>N2B|Wf!*3_ z7hW*j6p3TvpghEc6-wufFiY!%-GvOx*bZrhZu+7?iSrZL5q9}igiF^*R3%DE4aCHZ zqu>xS8LkW+Auv%z-<1Xs92u23R$nk@Pk}MU5!gT|c7vGlEA%G^2th&Q*zfg%-D^=f z&J_}jskj|Q;73NP4<4k*Y%pXPU2Thoqr+5uH1yEYM|VtBPW6lXaetokD0u z9qVek6Q&wk)tFbQ8(^HGf3Wp16gKmr>G;#G(HRBx?F`9AIRboK+;OfHaLJ(P>IP0w zyTbTkx_THEOs%Q&aPrxbZrJlio+hCC_HK<4%f3ZoSAyG7Dn`=X=&h@m*|UYO-4Hq0 z-Bq&+Ie!S##4A6OGoC~>ZW`Y5J)*ouaFl_e9GA*VSL!O_@xGiBw!AF}1{tB)z(w%c zS1Hmrb9OC8>0a_$BzeiN?rkPLc9%&;1CZW*4}CDDNr2gcl_3z+WC15&H1Zc2{o~i) z)LLW=WQ{?ricmC`G1GfJ0Yp4Dy~Ba;j6ZV4r{8xRs`13{dD!xXmr^Aga|C=iSmor% z8hi|pTXH)5Yf&v~exp3o+sY4B^^b*eYkkCYl*T{*=-0HniSA_1F53eCb{x~1k3*`W zr~};p1A`k{1DV9=UPnLDgz{aJH=-LQo<5%+Em!DNN252xwIf*wF_zS^!(XSm(9eoj z=*dXG&n0>)_)N5oc6v!>-bd(2ragD8O=M|wGW z!xJQS<)u70m&6OmrF0WSsr@I%T*c#Qo#Ha4d3COcX+9}hM5!7JIGF>7<~C(Ear^Sn zm^ZFkV6~Ula6+8S?oOROOA6$C&q&dp`>oR-2Ym3(HT@O7Sd5c~+kjrmM)YmgPH*tL zX+znN>`tv;5eOfX?h{AuX^LK~V#gPCu=)Tigtq9&?7Xh$qN|%A$?V*v=&-2F$zTUv z`C#WyIrChS5|Kgm_GeudCFf;)!WH7FI60j^0o#65o6`w*S7R@)88n$1nrgU(oU0M9 zx+EuMkC>(4j1;m6NoGqEkpJYJ?vc|B zOlwT3t&UgL!pX_P*6g36`ZXQ; z9~Cv}ANFnJGp(;ZhS(@FT;3e)0)Kp;h^x;$*xZn*k0U6-&FwI=uOGaODdrsp-!K$Ac32^c{+FhI-HkYd5v=`PGsg%6I`4d9Jy)uW0y%) zm&j^9WBAp*P8#kGJUhB!L?a%h$hJgQrx!6KCB_TRo%9{t0J7KW8!o1B!NC)VGLM5! zpZy5Jc{`r{1e(jd%jsG7k%I+m#CGS*BPA65ZVW~fLYw0dA-H_}O zrkGFL&P1PG9p2(%QiEWm6x;U-U&I#;Em$nx-_I^wtgw3xUPVVu zqSuKnx&dIT-XT+T10p;yjo1Y)z(x1fb8Dzfn8e yu?e%!_ptzGB|8GrCfu%p?(_ zQccdaaVK$5bz;*rnyK{_SQYM>;aES6Qs^lj9lEs6_J+%nIiuQC*fN;z8md>r_~Mfl zU%p5Dt_YT>gQqfr@`cR!$NWr~+`CZb%dn;WtzrAOI>P_JtsB76PYe*<%H(y>qx-`Kq!X_; z<{RpAqYhE=L1r*M)gNF3B8r(<%8mo*SR2hu zccLRZwGARt)Hlo1euqTyM>^!HK*!Q2P;4UYrysje@;(<|$&%vQekbn|0Ruu_Io(w4#%p6ld2Yp7tlA`Y$cciThP zKzNGIMPXX%&Ud0uQh!uQZz|FB`4KGD?3!ND?wQt6!n*f4EmCoJUh&b?;B{|lxs#F- z31~HQ`SF4x$&v00@(P+j1pAaj5!s`)b2RDBp*PB=2IB>oBF!*6vwr7Dp%zpAx*dPr zb@Zjq^XjN?O4QcZ*O+8>)|HlrR>oD*?WQl5ri3R#2?*W6iJ>>kH%KnnME&TT@ZzrHS$Q%LC?n|e>V+D+8D zYc4)QddFz7I8#}y#Wj6>4P%34dZH~OUDb?uP%-E zwjXM(?Sg~1!|wI(RVuxbu)-rH+O=igSho_pDCw(c6b=P zKk4ATlB?bj9+HHlh<_!&z0rx13K3ZrAR8W)!@Y}o`?a*JJsD+twZIv`W)@Y?Amu_u zz``@-e2X}27$i(2=9rvIu5uTUOVhzwu%mNazS|lZb&PT;XE2|B&W1>=B58#*!~D&) zfVmJGg8UdP*fx(>Cj^?yS^zH#o-$Q-*$SnK(ZVFkw+er=>N^7!)FtP3y~Xxnu^nzY zikgB>Nj0%;WOltWIob|}%lo?_C7<``a5hEkx&1ku$|)i>Rh6@3h*`slY=9U}(Ql_< zaNG*J8vb&@zpdhAvv`?{=zDedJ23TD&Zg__snRAH4eh~^oawdYi6A3w8<Ozh@Kw)#bdktM^GVb zrG08?0bG?|NG+w^&JvD*7LAbjED{_Zkc`3H!My>0u5Q}m!+6VokMLXxl`Mkd=g&Xx z-a>m*#G3SLlhbKB!)tnzfWOBV;u;ftU}S!NdD5+YtOjLg?X}dl>7m^gOpihrf1;PY zvll&>dIuUGs{Qnd- zwIR3oIrct8Va^Tm0t#(bJD7c$Z7DO9*7NnRZorrSm`b`cxz>OIC;jSE3DO8`hX955ui`s%||YQtt2 z5DNA&pG-V+4oI2s*x^>-$6J?p=I>C|9wZF8z;VjR??Icg?1w2v5Me+FgAeGGa8(3S z4vg*$>zC-WIVZtJ7}o9{D-7d>zCe|z#<9>CFve-OPAYsneTb^JH!Enaza#j}^mXy1 z+ULn^10+rWLF6j2>Ya@@Kq?26>AqK{A_| zQKb*~F1>sE*=d?A?W7N2j?L09_7n+HGi{VY;MoTGr_)G9)ot$p!-UY5zZ2Xtbm=t z@dpPSGwgH=QtIcEulQNI>S-#ifbnO5EWkI;$A|pxJd885oM+ zGZ0_0gDvG8q2xebj+fbCHYfAXuZStH2j~|d^sBAzo46(K8n59+T6rzBwK)^rfPT+B zyIFw)9YC-V^rhtK`!3jrhmW-sTmM+tPH+;nwjL#-SjQPUZ53L@A>y*rt(#M(qsiB2 zx6B)dI}6Wlsw%bJ8h|(lhkJVogQZA&n{?Vgs6gNSXzuZpEyu*xySy8ro07QZ7Vk1!3tJphN_5V7qOiyK8p z#@jcDD8nmtYi1^l8ml;AF<#IPK?!pqf9D4moYk>d99Im}Jtwj6c#+A;f)CQ*f-hZ< z=p_T86jog%!p)D&5g9taSwYi&eP z#JuEK%+NULWus;0w32-SYFku#i}d~+{Pkho&^{;RxzP&0!RCm3-9K6`>KZpnzS6?L z^H^V*s!8<>x8bomvD%rh>Zp3>Db%kyin;qtl+jAv8Oo~1g~mqGAC&Qi_wy|xEt2iz zWAJEfTV%cl2Cs<1L&DLRVVH05EDq`pH7Oh7sR`NNkL%wi}8n>IXcO40hp+J+sC!W?!krJf!GJNE8uj zg-y~Ns-<~D?yqbzVRB}G>0A^f0!^N7l=$m0OdZuqAOQqLc zX?AEGr1Ht+inZ-Qiwnl@Z0qukd__a!C*CKuGdy5#nD7VUBM^6OCpxCa2A(X;e0&V4 zM&WR8+wErQ7UIc6LY~Q9x%Sn*Tn>>P`^t&idaOEnOd(Ufw#>NoR^1QdhJ8s`h^|R_ zXX`c5*O~Xdvh%q;7L!_!ohf$NfEBmCde|#uVZvEo>OfEq%+Ns7&_f$OR9xsihRpBb z+cjk8LyDm@U{YN>+r46?nn{7Gh(;WhFw6GAxtcKD+YWV?uge>;+q#Xx4!GpRkVZYu zzsF}1)7$?%s9g9CH=Zs+B%M_)+~*j3L0&Q9u7!|+T`^O{xE6qvAP?XWv9_MrZKdo& z%IyU)$Q95AB4!#hT!_dA>4e@zjOBD*Y=XjtMm)V|+IXzjuM;(l+8aA5#Kaz_$rR6! zj>#&^DidYD$nUY(D$mH`9eb|dtV0b{S>H6FBfq>t5`;OxA4Nn{J(+XihF(stSche7$es&~N$epi&PDM_N`As;*9D^L==2Q7Z2zD+CiU(|+-kL*VG+&9!Yb3LgPy?A zm7Z&^qRG_JIxK7-FBzZI3Q<;{`DIxtc48k> zc|0dmX;Z=W$+)qE)~`yn6MdoJ4co;%!`ddy+FV538Y)j(vg}5*k(WK)KWZ3WaOG!8 z!syGn=s{H$odtpqFrT#JGM*utN7B((abXnpDM6w56nhw}OY}0TiTG1#f*VFZr+^-g zbP10`$LPq_;PvrA1XXlyx2uM^mrjTzX}w{yuLo-cOClE8MMk47T25G8M!9Z5ypOSV zAJUBGEg5L2fY)ZGJb^E34R2zJ?}Vf>{~gB!8=5Z) z9y$>5c)=;o0HeHHSuE4U)#vG&KF|I%-cF6f$~pdYJWk_dD}iOA>iA$O$+4%@>JU08 zS`ep)$XLPJ+n0_i@PkF#ri6T8?ZeAot$6JIYHm&P6EB=BiaNY|aA$W0I+nz*zkz_z zkEru!tj!QUffq%)8y0y`T&`fuus-1p>=^hnBiBqD^hXrPs`PY9tU3m0np~rISY09> z`P3s=-kt_cYcxWd{de@}TwSqg*xVhp;E9zCsnXo6z z?f&Sv^U7n4`xr=mXle94HzOdN!2kB~4=%)u&N!+2;z6UYKUDqi-s6AZ!haB;@&B`? z_TRX0%@suz^TRdCb?!vNJYPY8L_}&07uySH9%W^Tc&1pia6y1q#?*Drf}GjGbPjBS zbOPcUY#*$3sL2x4v_i*Y=N7E$mR}J%|GUI(>WEr+28+V z%v5{#e!UF*6~G&%;l*q*$V?&r$Pp^sE^i-0$+RH3ERUUdQ0>rAq2(2QAbG}$y{de( z>{qD~GGuOk559Y@%$?N^1ApVL_a704>8OD%8Y%8B;FCt%AoPu8*D1 zLB5X>b}Syz81pn;xnB}%0FnwazlWfUV)Z-~rZg6~b z6!9J$EcE&sEbzcy?CI~=boWA&eeIa%z(7SE^qgVLz??1Vbc1*aRvc%Mri)AJaAG!p z$X!_9Ds;Zz)f+;%s&dRcJt2==P{^j3bf0M=nJd&xwUGlUFn?H=2W(*2I2Gdu zv!gYCwM10aeus)`RIZSrCK=&oKaO_Ry~D1B5!y0R=%!i2*KfXGYX&gNv_u+n9wiR5 z*e$Zjju&ODRW3phN925%S(jL+bCHv6rZtc?!*`1TyYXT6%Ju=|X;6D@lq$8T zW{Y|e39ioPez(pBH%k)HzFITXHvnD6hw^lIoUMA;qAJ^CU?top1fo@s7xT13Fvn1H z6JWa-6+FJF#x>~+A;D~;VDs26>^oH0EI`IYT2iagy23?nyJ==i{g4%HrAf1-*v zK1)~@&(KkwR7TL}L(A@C_S0G;-GMDy=MJn2$FP5s<%wC)4jC5PXoxrQBFZ_k0P{{s@sz+gX`-!=T8rcB(=7vW}^K6oLWMmp(rwDh}b zwaGGd>yEy6fHv%jM$yJXo5oMAQ>c9j`**}F?MCry;T@47@r?&sKHgVe$MCqk#Z_3S z1GZI~nOEN*P~+UaFGnj{{Jo@16`(qVNtbU>O0Hf57-P>x8Jikp=`s8xWs^dAJ9lCQ z)GFm+=OV%AMVqVATtN@|vp61VVAHRn87}%PC^RAzJ%JngmZTasWBAWsoAqBU+8L8u z4A&Pe?fmTm0?mK-BL9t+{y7o(7jm+RpOhL9KnY#E&qu^}B6=K_dB}*VlSEiC9fn)+V=J;OnN)Ta5v66ic1rG+dGAJ1 z1%Zb_+!$=tQ~lxQrzv3x#CPb?CekEkA}0MYSgx$Jdd}q8+R=ma$|&1a#)TQ=l$1tQ z=tL9&_^vJ)Pk}EDO-va`UCT1m#Uty1{v^A3P~83_#v^ozH}6*9mIjIr;t3Uv%@VeW zGL6(CwCUp)Jq%G0bIG%?{_*Y#5IHf*5M@wPo6A{$Um++Co$wLC=J1aoG93&T7Ho}P z=mGEPP7GbvoG!uD$k(H3A$Z))+i{Hy?QHdk>3xSBXR0j!11O^mEe9RHmw!pvzv?Ua~2_l2Yh~_!s1qS`|0~0)YsbHSz8!mG)WiJE| z2f($6TQtt6L_f~ApQYQKSb=`053LgrQq7G@98#igV>y#i==-nEjQ!XNu9 z~;mE+gtj4IDDNQJ~JVk5Ux6&LCSFL!y=>79kE9=V}J7tD==Ga+IW zX)r7>VZ9dY=V&}DR))xUoV!u(Z|%3ciQi_2jl}3=$Agc(`RPb z8kEBpvY>1FGQ9W$n>Cq=DIpski};nE)`p3IUw1Oz0|wxll^)4dq3;CCY@RyJgFgc# zKouFh!`?Xuo{IMz^xi-h=StCis_M7yq$u) z?XHvw*HP0VgR+KR6wI)jEMX|ssqYvSf*_3W8zVTQzD?3>H!#>InzpSO)@SC8q*ii- z%%h}_#0{4JG;Jm`4zg};BPTGkYamx$Xo#O~lBirRY)q=5M45n{GCfV7h9qwyu1NxOMoP4)jjZMxmT|IQQh0U7C$EbnMN<3)Kk?fFHYq$d|ICu>KbY_hO zTZM+uKHe(cIZfEqyzyYSUBZa8;Fcut-GN!HSA9ius`ltNebF46ZX_BbZNU}}ZOm{M2&nANL9@0qvih15(|`S~z}m&h!u4x~(%MAO$jHRWNfuxWF#B)E&g3ghSQ9|> z(MFaLQj)NE0lowyjvg8z0#m6FIuKE9lDO~Glg}nSb7`~^&#(Lw{}GVOS>U)m8bF}x zVjbXljBm34Cs-yM6TVusr+3kYFjr28STT3g056y3cH5Tmge~ASxBj z%|yb>$eF;WgrcOZf569sDZOVwoo%8>XO>XQOX1OyN9I-SQgrm;U;+#3OI(zrWyow3 zk==|{lt2xrQ%FIXOTejR>;wv(Pb8u8}BUpx?yd(Abh6? zsoO3VYWkeLnF43&@*#MQ9-i-d0t*xN-UEyNKeyNMHw|A(k(_6QKO=nKMCxD(W(Yop zsRQ)QeL4X3Lxp^L%wzi2-WVSsf61dqliPUM7srDB?Wm6Lzn0&{*}|IsKQW;02(Y&| zaTKv|`U(pSzuvR6Rduu$wzK_W-Y-7>7s?G$)U}&uK;<>vU}^^ns@Z!p+9?St1s)dG zK%y6xkPyyS1$~&6v{kl?Md6gwM|>mt6Upm>oa8RLD^8T{0?HC!Z>;(Bob7el(DV6x zi`I)$&E&ngwFS@bi4^xFLAn`=fzTC;aimE^!cMI2n@Vo%Ae-ne`RF((&5y6xsjjAZ zVguVoQ?Z9uk$2ON;ersE%PU*xGO@T*;j1BO5#TuZKEf(mB7|g7pcEA=nYJ{s3vlbg zd4-DUlD{*6o%Gc^N!Nptgay>j6E5;3psI+C3Q!1ZIbeCubW%w4pq9)MSDyB{HLm|k zxv-{$$A*pS@csolri$Ge<4VZ}e~78JOL-EVyrbxKra^d{?|NnPp86!q>t<&IP07?Z z^>~IK^k#OEKgRH+LjllZXk7iA>2cfH6+(e&9ku5poo~6y{GC5>(bRK7hwjiurqAiZ zg*DmtgY}v83IjE&AbiWgMyFbaRUPZ{lYiz$U^&Zt2YjG<%m((&_JUbZcfJ22(>bi5 z!J?<7AySj0JZ&<-qXX;mcV!f~>G=sB0KnjWca4}vrtunD^1TrpfeS^4dvFr!65knK zZh`d;*VOkPs4*-9kL>$GP0`(M!j~B;#x?Ba~&s6CopvO86oM?-? zOw#dIRc;6A6T?B`Qp%^<U5 z19x(ywSH$_N+Io!6;e?`tWaM$`=Db!gzx|lQ${DG!zb1Zl&|{kX0y6xvO1o z220r<-oaS^^R2pEyY;=Qllqpmue|5yI~D|iI!IGt@iod{Opz@*ml^w2bNs)p`M(Io z|E;;m*Xpjd9l)4G#KaWfV(t8YUn@A;nK^#xgv=LtnArX|vWQVuw3}B${h+frU2>9^ z!l6)!Uo4`5k`<<;E(ido7M6lKTgWezNLq>U*=uz&s=cc$1%>VrAeOoUtA|T6gO4>UNqsdK=NF*8|~*sl&wI=x9-EGiq*aqV!(VVXA57 zw9*o6Ir8Lj1npUXvlevtn(_+^X5rzdR>#(}4YcB9O50q97%rW2me5_L=%ffYPUSRc z!vv?Kv>dH994Qi>U(a<0KF6NH5b16enCp+mw^Hb3Xs1^tThFpz!3QuN#}KBbww`(h z7GO)1olDqy6?T$()R7y%NYx*B0k_2IBiZ14&8|JPFxeMF{vSTxF-Vi3+ZOI=Thq2} zyQgjYY1_7^ZQHh{?P))4+qUiQJLi1&{yE>h?~jU%tjdV0h|FENbM3X(KnJdPKc?~k zh=^Ixv*+smUll!DTWH!jrV*wSh*(mx0o6}1@JExzF(#9FXgmTXVoU+>kDe68N)dkQ zH#_98Zv$}lQwjKL@yBd;U(UD0UCl322=pav<=6g>03{O_3oKTq;9bLFX1ia*lw;#K zOiYDcBJf)82->83N_Y(J7Kr_3lE)hAu;)Q(nUVydv+l+nQ$?|%MWTy`t>{havFSQloHwiIkGK9YZ79^9?AZo0ZyQlVR#}lF%dn5n%xYksXf8gnBm=wO7g_^! zauQ-bH1Dc@3ItZ-9D_*pH}p!IG7j8A_o94#~>$LR|TFq zZ-b00*nuw|-5C2lJDCw&8p5N~Z1J&TrcyErds&!l3$eSz%`(*izc;-?HAFD9AHb-| z>)id`QCrzRws^9(#&=pIx9OEf2rmlob8sK&xPCWS+nD~qzU|qG6KwA{zbikcfQrdH z+ zQg>O<`K4L8rN7`GJB0*3<3`z({lWe#K!4AZLsI{%z#ja^OpfjU{!{)x0ZH~RB0W5X zTwN^w=|nA!4PEU2=LR05x~}|B&ZP?#pNgDMwD*ajI6oJqv!L81gu=KpqH22avXf0w zX3HjbCI!n9>l046)5rr5&v5ja!xkKK42zmqHzPx$9Nn_MZk`gLeSLgC=LFf;H1O#B zn=8|^1iRrujHfbgA+8i<9jaXc;CQBAmQvMGQPhFec2H1knCK2x!T`e6soyrqCamX% zTQ4dX_E*8so)E*TB$*io{$c6X)~{aWfaqdTh=xEeGvOAN9H&-t5tEE-qso<+C!2>+ zskX51H-H}#X{A75wqFe-J{?o8Bx|>fTBtl&tcbdR|132Ztqu5X0i-pisB-z8n71%q%>EF}yy5?z=Ve`}hVh{Drv1YWL zW=%ug_&chF11gDv3D6B)Tz5g54H0mDHNjuKZ+)CKFk4Z|$RD zfRuKLW`1B>B?*RUfVd0+u8h3r-{@fZ{k)c!93t1b0+Q9vOaRnEn1*IL>5Z4E4dZ!7 ztp4GP-^1d>8~LMeb}bW!(aAnB1tM_*la=Xx)q(I0Y@__Zd$!KYb8T2VBRw%e$iSdZ zkwdMwd}eV9q*;YvrBFTv1>1+}{H!JK2M*C|TNe$ZSA>UHKk);wz$(F$rXVc|sI^lD zV^?_J!3cLM;GJuBMbftbaRUs$;F}HDEDtIeHQ)^EJJ1F9FKJTGH<(Jj`phE6OuvE) zqK^K`;3S{Y#1M@8yRQwH`?kHMq4tHX#rJ>5lY3DM#o@or4&^_xtBC(|JpGTfrbGkA z2Tu+AyT^pHannww!4^!$5?@5v`LYy~T`qs7SYt$JgrY(w%C+IWA;ZkwEF)u5sDvOK zGk;G>Mh&elvXDcV69J_h02l&O;!{$({fng9Rlc3ID#tmB^FIG^w{HLUpF+iB`|
NnX)EH+Nua)3Y(c z&{(nX_ht=QbJ%DzAya}!&uNu!4V0xI)QE$SY__m)SAKcN0P(&JcoK*Lxr@P zY&P=}&B3*UWNlc|&$Oh{BEqwK2+N2U$4WB7Fd|aIal`FGANUa9E-O)!gV`((ZGCc$ zBJA|FFrlg~9OBp#f7aHodCe{6= zay$6vN~zj1ddMZ9gQ4p32(7wD?(dE>KA2;SOzXRmPBiBc6g`eOsy+pVcHu=;Yd8@{ zSGgXf@%sKKQz~;!J;|2fC@emm#^_rnO0esEn^QxXgJYd`#FPWOUU5b;9eMAF zZhfiZb|gk8aJIw*YLp4!*(=3l8Cp{(%p?ho22*vN9+5NLV0TTazNY$B5L6UKUrd$n zjbX%#m7&F#U?QNOBXkiiWB*_tk+H?N3`vg;1F-I+83{M2!8<^nydGr5XX}tC!10&e z7D36bLaB56WrjL&HiiMVtpff|K%|*{t*ltt^5ood{FOG0<>k&1h95qPio)2`eL${YAGIx(b4VN*~nKn6E~SIQUuRH zQ+5zP6jfnP$S0iJ@~t!Ai3o`X7biohli;E zT#yXyl{bojG@-TGZzpdVDXhbmF%F9+-^YSIv|MT1l3j zrxOFq>gd2%U}?6}8mIj?M zc077Zc9fq(-)4+gXv?Az26IO6eV`RAJz8e3)SC7~>%rlzDwySVx*q$ygTR5kW2ds- z!HBgcq0KON9*8Ff$X0wOq$`T7ml(@TF)VeoF}x1OttjuVHn3~sHrMB++}f7f9H%@f z=|kP_?#+fve@{0MlbkC9tyvQ_R?lRdRJ@$qcB(8*jyMyeME5ns6ypVI1Xm*Zr{DuS zZ!1)rQfa89c~;l~VkCiHI|PCBd`S*2RLNQM8!g9L6?n`^evQNEwfO@&JJRme+uopQX0%Jo zgd5G&#&{nX{o?TQwQvF1<^Cg3?2co;_06=~Hcb6~4XWpNFL!WU{+CK;>gH%|BLOh7@!hsa(>pNDAmpcuVO-?;Bic17R}^|6@8DahH)G z!EmhsfunLL|3b=M0MeK2vqZ|OqUqS8npxwge$w-4pFVXFq$_EKrZY?BuP@Az@(k`L z`ViQBSk`y+YwRT;&W| z2e3UfkCo^uTA4}Qmmtqs+nk#gNr2W4 zTH%hhErhB)pkXR{B!q5P3-OM+M;qu~f>}IjtF%>w{~K-0*jPVLl?Chz&zIdxp}bjx zStp&Iufr58FTQ36AHU)0+CmvaOpKF;W@sMTFpJ`j;3d)J_$tNQI^c<^1o<49Z(~K> z;EZTBaVT%14(bFw2ob@?JLQ2@(1pCdg3S%E4*dJ}dA*v}_a4_P(a`cHnBFJxNobAv zf&Zl-Yt*lhn-wjZsq<9v-IsXxAxMZ58C@e0!rzhJ+D@9^3~?~yllY^s$?&oNwyH!#~6x4gUrfxplCvK#!f z$viuszW>MFEcFL?>ux*((!L$;R?xc*myjRIjgnQX79@UPD$6Dz0jutM@7h_pq z0Zr)#O<^y_K6jfY^X%A-ip>P%3saX{!v;fxT-*0C_j4=UMH+Xth(XVkVGiiKE#f)q z%Jp=JT)uy{&}Iq2E*xr4YsJ5>w^=#-mRZ4vPXpI6q~1aFwi+lQcimO45V-JXP;>(Q zo={U`{=_JF`EQj87Wf}{Qy35s8r1*9Mxg({CvOt}?Vh9d&(}iI-quvs-rm~P;eRA@ zG5?1HO}puruc@S{YNAF3vmUc2B4!k*yi))<5BQmvd3tr}cIs#9)*AX>t`=~{f#Uz0 z0&Nk!7sSZwJe}=)-R^$0{yeS!V`Dh7w{w5rZ9ir!Z7Cd7dwZcK;BT#V0bzTt>;@Cl z#|#A!-IL6CZ@eHH!CG>OO8!%G8&8t4)Ro@}USB*k>oEUo0LsljsJ-%5Mo^MJF2I8- z#v7a5VdJ-Cd%(a+y6QwTmi+?f8Nxtm{g-+WGL>t;s#epv7ug>inqimZCVm!uT5Pf6 ziEgQt7^%xJf#!aPWbuC_3Nxfb&CFbQy!(8ANpkWLI4oSnH?Q3f?0k1t$3d+lkQs{~(>06l&v|MpcFsyAv zin6N!-;pggosR*vV=DO(#+}4ps|5$`udE%Kdmp?G7B#y%H`R|i8skKOd9Xzx8xgR$>Zo2R2Ytktq^w#ul4uicxW#{ zFjG_RNlBroV_n;a7U(KIpcp*{M~e~@>Q#Av90Jc5v%0c>egEdY4v3%|K1XvB{O_8G zkTWLC>OZKf;XguMH2-Pw{BKbFzaY;4v2seZV0>^7Q~d4O=AwaPhP3h|!hw5aqOtT@ z!SNz}$of**Bl3TK209@F=Tn1+mgZa8yh(Png%Zd6Mt}^NSjy)etQrF zme*llAW=N_8R*O~d2!apJnF%(JcN??=`$qs3Y+~xs>L9x`0^NIn!8mMRFA_tg`etw z3k{9JAjnl@ygIiJcNHTy02GMAvBVqEss&t2<2mnw!; zU`J)0>lWiqVqo|ex7!+@0i>B~BSU1A_0w#Ee+2pJx0BFiZ7RDHEvE*ptc9md(B{&+ zKE>TM)+Pd>HEmdJao7U@S>nL(qq*A)#eLOuIfAS@j`_sK0UEY6OAJJ-kOrHG zjHx`g!9j*_jRcJ%>CE9K2MVf?BUZKFHY?EpV6ai7sET-tqk=nDFh-(65rhjtlKEY% z@G&cQ<5BKatfdA1FKuB=i>CCC5(|9TMW%K~GbA4}80I5%B}(gck#Wlq@$nO3%@QP_ z8nvPkJFa|znk>V92cA!K1rKtr)skHEJD;k8P|R8RkCq1Rh^&}Evwa4BUJz2f!2=MH zo4j8Y$YL2313}H~F7@J7mh>u%556Hw0VUOz-Un@ZASCL)y8}4XXS`t1AC*^>PLwIc zUQok5PFS=*#)Z!3JZN&eZ6ZDP^-c@StY*t20JhCnbMxXf=LK#;`4KHEqMZ-Ly9KsS zI2VUJGY&PmdbM+iT)zek)#Qc#_i4uH43 z@T5SZBrhNCiK~~esjsO9!qBpaWK<`>!-`b71Y5ReXQ4AJU~T2Njri1CEp5oKw;Lnm)-Y@Z3sEY}XIgSy%xo=uek(kAAH5MsV$V3uTUsoTzxp_rF=tx zV07vlJNKtJhCu`b}*#m&5LV4TAE&%KtHViDAdv#c^x`J7bg z&N;#I2GkF@SIGht6p-V}`!F_~lCXjl1BdTLIjD2hH$J^YFN`7f{Q?OHPFEM$65^!u zNwkelo*5+$ZT|oQ%o%;rBX$+?xhvjb)SHgNHE_yP%wYkkvXHS{Bf$OiKJ5d1gI0j< zF6N}Aq=(WDo(J{e-uOecxPD>XZ@|u-tgTR<972`q8;&ZD!cep^@B5CaqFz|oU!iFj zU0;6fQX&~15E53EW&w1s9gQQ~Zk16X%6 zjG`j0yq}4deX2?Tr(03kg>C(!7a|b9qFI?jcE^Y>-VhudI@&LI6Qa}WQ>4H_!UVyF z((cm&!3gmq@;BD#5P~0;_2qgZhtJS|>WdtjY=q zLnHH~Fm!cxw|Z?Vw8*~?I$g#9j&uvgm7vPr#&iZgPP~v~BI4jOv;*OQ?jYJtzO<^y z7-#C={r7CO810!^s(MT!@@Vz_SVU)7VBi(e1%1rvS!?PTa}Uv`J!EP3s6Y!xUgM^8 z4f!fq<3Wer_#;u!5ECZ|^c1{|q_lh3m^9|nsMR1#Qm|?4Yp5~|er2?W^7~cl;_r4WSme_o68J9p03~Hc%X#VcX!xAu%1`R!dfGJCp zV*&m47>s^%Ib0~-2f$6oSgn3jg8m%UA;ArcdcRyM5;}|r;)?a^D*lel5C`V5G=c~k zy*w_&BfySOxE!(~PI$*dwG><+-%KT5p?whOUMA*k<9*gi#T{h3DAxzAPxN&Xws8o9Cp*`PA5>d9*Z-ynV# z9yY*1WR^D8|C%I@vo+d8r^pjJ$>eo|j>XiLWvTWLl(^;JHCsoPgem6PvegHb-OTf| zvTgsHSa;BkbG=(NgPO|CZu9gUCGr$8*EoH2_Z#^BnxF0yM~t`|9ws_xZ8X8iZYqh! zAh;HXJ)3P&)Q0(&F>!LN0g#bdbis-cQxyGn9Qgh`q+~49Fqd2epikEUw9caM%V6WgP)532RMRW}8gNS%V%Hx7apSz}tn@bQy!<=lbhmAH=FsMD?leawbnP5BWM0 z5{)@EEIYMu5;u)!+HQWhQ;D3_Cm_NADNeb-f56}<{41aYq8p4=93d=-=q0Yx#knGYfXVt z+kMxlus}t2T5FEyCN~!}90O_X@@PQpuy;kuGz@bWft%diBTx?d)_xWd_-(!LmVrh**oKg!1CNF&LX4{*j|) zIvjCR0I2UUuuEXh<9}oT_zT#jOrJAHNLFT~Ilh9hGJPI1<5`C-WA{tUYlyMeoy!+U zhA#=p!u1R7DNg9u4|QfED-2TuKI}>p#2P9--z;Bbf4Op*;Q9LCbO&aL2i<0O$ByoI z!9;Ght733FC>Pz>$_mw(F`zU?`m@>gE`9_p*=7o=7av`-&ifU(^)UU`Kg3Kw`h9-1 z6`e6+im=|m2v`pN(2dE%%n8YyQz;#3Q-|x`91z?gj68cMrHl}C25|6(_dIGk*8cA3 zRHB|Nwv{@sP4W+YZM)VKI>RlB`n=Oj~Rzx~M+Khz$N$45rLn6k1nvvD^&HtsMA4`s=MmuOJID@$s8Ph4E zAmSV^+s-z8cfv~Yd(40Sh4JG#F~aB>WFoX7ykaOr3JaJ&Lb49=B8Vk-SQT9%7TYhv z?-Pprt{|=Y5ZQ1?od|A<_IJU93|l4oAfBm?3-wk{O<8ea+`}u%(kub(LFo2zFtd?4 zwpN|2mBNywv+d^y_8#<$r>*5+$wRTCygFLcrwT(qc^n&@9r+}Kd_u@Ithz(6Qb4}A zWo_HdBj#V$VE#l6pD0a=NfB0l^6W^g`vm^sta>Tly?$E&{F?TTX~DsKF~poFfmN%2 z4x`Dc{u{Lkqz&y!33;X}weD}&;7p>xiI&ZUb1H9iD25a(gI|`|;G^NwJPv=1S5e)j z;U;`?n}jnY6rA{V^ zxTd{bK)Gi^odL3l989DQlN+Zs39Xe&otGeY(b5>rlIqfc7Ap4}EC?j<{M=hlH{1+d zw|c}}yx88_xQr`{98Z!d^FNH77=u(p-L{W6RvIn40f-BldeF-YD>p6#)(Qzf)lfZj z?3wAMtPPp>vMehkT`3gToPd%|D8~4`5WK{`#+}{L{jRUMt zrFz+O$C7y8$M&E4@+p+oV5c%uYzbqd2Y%SSgYy#xh4G3hQv>V*BnuKQhBa#=oZB~w{azUB+q%bRe_R^ z>fHBilnRTUfaJ201czL8^~Ix#+qOHSO)A|xWLqOxB$dT2W~)e-r9;bm=;p;RjYahB z*1hegN(VKK+ztr~h1}YP@6cfj{e#|sS`;3tJhIJK=tVJ-*h-5y9n*&cYCSdg#EHE# zSIx=r#qOaLJoVVf6v;(okg6?*L_55atl^W(gm^yjR?$GplNP>BZsBYEf_>wM0Lc;T zhf&gpzOWNxS>m+mN92N0{;4uw`P+9^*|-1~$uXpggj4- z^SFc4`uzj2OwdEVT@}Q`(^EcQ_5(ZtXTql*yGzdS&vrS_w>~~ra|Nb5abwf}Y!uq6R5f&6g2ge~2p(%c< z@O)cz%%rr4*cRJ5f`n@lvHNk@lE1a*96Kw6lJ~B-XfJW%?&-y?;E&?1AacU@`N`!O z6}V>8^%RZ7SQnZ-z$(jsX`amu*5Fj8g!3RTRwK^`2_QHe;_2y_n|6gSaGyPmI#kA0sYV<_qOZc#-2BO%hX)f$s-Z3xlI!ub z^;3ru11DA`4heAu%}HIXo&ctujzE2!6DIGE{?Zs>2}J+p&C$rc7gJC35gxhflorvsb%sGOxpuWhF)dL_&7&Z99=5M0b~Qa;Mo!j&Ti_kXW!86N%n= zSC@6Lw>UQ__F&+&Rzv?gscwAz8IP!n63>SP)^62(HK98nGjLY2*e^OwOq`3O|C92? z;TVhZ2SK%9AGW4ZavTB9?)mUbOoF`V7S=XM;#3EUpR+^oHtdV!GK^nXzCu>tpR|89 zdD{fnvCaN^^LL%amZ^}-E+214g&^56rpdc@yv0b<3}Ys?)f|fXN4oHf$six)-@<;W&&_kj z-B}M5U*1sb4)77aR=@%I?|Wkn-QJVuA96an25;~!gq(g1@O-5VGo7y&E_srxL6ZfS z*R%$gR}dyONgju*D&?geiSj7SZ@ftyA|}(*Y4KbvU!YLsi1EDQQCnb+-cM=K1io78o!v*);o<XwjaQH%)uIP&Zm?)Nfbfn;jIr z)d#!$gOe3QHp}2NBak@yYv3m(CPKkwI|{;d=gi552u?xj9ObCU^DJFQp4t4e1tPzM zvsRIGZ6VF+{6PvqsplMZWhz10YwS={?`~O0Ec$`-!klNUYtzWA^f9m7tkEzCy<_nS z=&<(awFeZvt51>@o_~>PLs05CY)$;}Oo$VDO)?l-{CS1Co=nxjqben*O1BR>#9`0^ zkwk^k-wcLCLGh|XLjdWv0_Hg54B&OzCE^3NCP}~OajK-LuRW53CkV~Su0U>zN%yQP zH8UH#W5P3-!ToO-2k&)}nFe`t+mdqCxxAHgcifup^gKpMObbox9LFK;LP3}0dP-UW z?Zo*^nrQ6*$FtZ(>kLCc2LY*|{!dUn$^RW~m9leoF|@Jy|M5p-G~j%+P0_#orRKf8 zvuu5<*XO!B?1E}-*SY~MOa$6c%2cM+xa8}_8x*aVn~57v&W(0mqN1W`5a7*VN{SUH zXz98DDyCnX2EPl-`Lesf`=AQT%YSDb`$%;(jUTrNen$NPJrlpPDP}prI>Ml!r6bCT;mjsg@X^#&<}CGf0JtR{Ecwd&)2zuhr#nqdgHj+g2n}GK9CHuwO zk>oZxy{vcOL)$8-}L^iVfJHAGfwN$prHjYV0ju}8%jWquw>}_W6j~m<}Jf!G?~r5&Rx)!9JNX!ts#SGe2HzobV5); zpj@&`cNcO&q+%*<%D7za|?m5qlmFK$=MJ_iv{aRs+BGVrs)98BlN^nMr{V_fcl_;jkzRju+c-y?gqBC_@J0dFLq-D9@VN&-`R9U;nv$Hg?>$oe4N&Ht$V_(JR3TG^! zzJsbQbi zFE6-{#9{G{+Z}ww!ycl*7rRdmU#_&|DqPfX3CR1I{Kk;bHwF6jh0opI`UV2W{*|nn zf_Y@%wW6APb&9RrbEN=PQRBEpM(N1w`81s=(xQj6 z-eO0k9=Al|>Ej|Mw&G`%q8e$2xVz1v4DXAi8G};R$y)ww638Y=9y$ZYFDM$}vzusg zUf+~BPX>(SjA|tgaFZr_e0{)+z9i6G#lgt=F_n$d=beAt0Sa0a7>z-?vcjl3e+W}+ z1&9=|vC=$co}-Zh*%3588G?v&U7%N1Qf-wNWJ)(v`iO5KHSkC5&g7CrKu8V}uQGcfcz zmBz#Lbqwqy#Z~UzHgOQ;Q-rPxrRNvl(&u6ts4~0=KkeS;zqURz%!-ERppmd%0v>iRlEf+H$yl{_8TMJzo0 z>n)`On|7=WQdsqhXI?#V{>+~}qt-cQbokEbgwV3QvSP7&hK4R{Z{aGHVS3;+h{|Hz z6$Js}_AJr383c_+6sNR|$qu6dqHXQTc6?(XWPCVZv=)D#6_;D_8P-=zOGEN5&?~8S zl5jQ?NL$c%O)*bOohdNwGIKM#jSAC?BVY={@A#c9GmX0=T(0G}xs`-%f3r=m6-cpK z!%waekyAvm9C3%>sixdZj+I(wQlbB4wv9xKI*T13DYG^T%}zZYJ|0$Oj^YtY+d$V$ zAVudSc-)FMl|54n=N{BnZTM|!>=bhaja?o7s+v1*U$!v!qQ%`T-6fBvmdPbVmro&d zk07TOp*KuxRUSTLRrBj{mjsnF8`d}rMViY8j`jo~Hp$fkv9F_g(jUo#Arp;Xw0M$~ zRIN!B22~$kx;QYmOkos@%|5k)!QypDMVe}1M9tZfkpXKGOxvKXB!=lo`p?|R1l=tA zp(1}c6T3Fwj_CPJwVsYtgeRKg?9?}%oRq0F+r+kdB=bFUdVDRPa;E~~>2$w}>O>v=?|e>#(-Lyx?nbg=ckJ#5U6;RT zNvHhXk$P}m9wSvFyU3}=7!y?Y z=fg$PbV8d7g25&-jOcs{%}wTDKm>!Vk);&rr;O1nvO0VrU&Q?TtYVU=ir`te8SLlS zKSNmV=+vF|ATGg`4$N1uS|n??f}C_4Sz!f|4Ly8#yTW-FBfvS48Tef|-46C(wEO_%pPhUC5$-~Y?!0vFZ^Gu`x=m7X99_?C-`|h zfmMM&Y@zdfitA@KPw4Mc(YHcY1)3*1xvW9V-r4n-9ZuBpFcf{yz+SR{ zo$ZSU_|fgwF~aakGr(9Be`~A|3)B=9`$M-TWKipq-NqRDRQc}ABo*s_5kV%doIX7LRLRau_gd@Rd_aLFXGSU+U?uAqh z8qusWWcvgQ&wu{|sRXmv?sl=xc<$6AR$+cl& zFNh5q1~kffG{3lDUdvEZu5c(aAG~+64FxdlfwY^*;JSS|m~CJusvi-!$XR`6@XtY2 znDHSz7}_Bx7zGq-^5{stTRy|I@N=>*y$zz>m^}^{d&~h;0kYiq8<^Wq7Dz0w31ShO^~LUfW6rfitR0(=3;Uue`Y%y@ex#eKPOW zO~V?)M#AeHB2kovn1v=n^D?2{2jhIQd9t|_Q+c|ZFaWt+r&#yrOu-!4pXAJuxM+Cx z*H&>eZ0v8Y`t}8{TV6smOj=__gFC=eah)mZt9gwz>>W$!>b3O;Rm^Ig*POZP8Rl0f zT~o=Nu1J|lO>}xX&#P58%Yl z83`HRs5#32Qm9mdCrMlV|NKNC+Z~ z9OB8xk5HJ>gBLi+m@(pvpw)1(OaVJKs*$Ou#@Knd#bk+V@y;YXT?)4eP9E5{J%KGtYinNYJUH9PU3A}66c>Xn zZ{Bn0<;8$WCOAL$^NqTjwM?5d=RHgw3!72WRo0c;+houoUA@HWLZM;^U$&sycWrFd zE7ekt9;kb0`lps{>R(}YnXlyGY}5pPd9zBpgXeJTY_jwaJGSJQC#-KJqmh-;ad&F- z-Y)E>!&`Rz!HtCz>%yOJ|v(u7P*I$jqEY3}(Z-orn4 zlI?CYKNl`6I){#2P1h)y(6?i;^z`N3bxTV%wNvQW+eu|x=kbj~s8rhCR*0H=iGkSj zk23lr9kr|p7#qKL=UjgO`@UnvzU)`&fI>1Qs7ubq{@+lK{hH* zvl6eSb9%yngRn^T<;jG1SVa)eA>T^XX=yUS@NCKpk?ovCW1D@!=@kn;l_BrG;hOTC z6K&H{<8K#dI(A+zw-MWxS+~{g$tI7|SfP$EYKxA}LlVO^sT#Oby^grkdZ^^lA}uEF zBSj$weBJG{+Bh@Yffzsw=HyChS(dtLE3i*}Zj@~!_T-Ay7z=B)+*~3|?w`Zd)Co2t zC&4DyB!o&YgSw+fJn6`sn$e)29`kUwAc+1MND7YjV%lO;H2}fNy>hD#=gT ze+-aFNpyKIoXY~Vq-}OWPBe?Rfu^{ps8>Xy%42r@RV#*QV~P83jdlFNgkPN=T|Kt7 zV*M`Rh*30&AWlb$;ae130e@}Tqi3zx2^JQHpM>j$6x`#{mu%tZlwx9Gj@Hc92IuY* zarmT|*d0E~vt6<+r?W^UW0&#U&)8B6+1+;k^2|FWBRP9?C4Rk)HAh&=AS8FS|NQaZ z2j!iZ)nbEyg4ZTp-zHwVlfLC~tXIrv(xrP8PAtR{*c;T24ycA-;auWsya-!kF~CWZ zw_uZ|%urXgUbc@x=L=_g@QJ@m#5beS@6W195Hn7>_}z@Xt{DIEA`A&V82bc^#!q8$ zFh?z_Vn|ozJ;NPd^5uu(9tspo8t%&-U9Ckay-s@DnM*R5rtu|4)~e)`z0P-sy?)kc zs_k&J@0&0!q4~%cKL)2l;N*T&0;mqX5T{Qy60%JtKTQZ-xb%KOcgqwJmb%MOOKk7N zgq})R_6**{8A|6H?fO+2`#QU)p$Ei2&nbj6TpLSIT^D$|`TcSeh+)}VMb}LmvZ{O| ze*1IdCt3+yhdYVxcM)Q_V0bIXLgr6~%JS<<&dxIgfL=Vnx4YHuU@I34JXA|+$_S3~ zy~X#gO_X!cSs^XM{yzDGNM>?v(+sF#<0;AH^YrE8smx<36bUsHbN#y57K8WEu(`qHvQ6cAZPo=J5C(lSmUCZ57Rj6cx!e^rfaI5%w}unz}4 zoX=nt)FVNV%QDJH`o!u9olLD4O5fl)xp+#RloZlaA92o3x4->?rB4`gS$;WO{R;Z3>cG3IgFX2EA?PK^M}@%1%A;?f6}s&CV$cIyEr#q5;yHdNZ9h{| z-=dX+a5elJoDo?Eq&Og!nN6A)5yYpnGEp}?=!C-V)(*~z-+?kY1Q7qs#Rsy%hu_60rdbB+QQNr?S1 z?;xtjUv|*E3}HmuNyB9aFL5H~3Ho0UsmuMZELp1a#CA1g`P{-mT?BchuLEtK}!QZ=3AWakRu~?f9V~3F;TV`5%9Pcs_$gq&CcU}r8gOO zC2&SWPsSG{&o-LIGTBqp6SLQZPvYKp$$7L4WRRZ0BR$Kf0I0SCFkqveCp@f)o8W)! z$%7D1R`&j7W9Q9CGus_)b%+B#J2G;l*FLz#s$hw{BHS~WNLODV#(!u_2Pe&tMsq={ zdm7>_WecWF#D=?eMjLj=-_z`aHMZ=3_-&E8;ibPmM}61i6J3is*=dKf%HC>=xbj4$ zS|Q-hWQ8T5mWde6h@;mS+?k=89?1FU<%qH9B(l&O>k|u_aD|DY*@~(`_pb|B#rJ&g zR0(~(68fpUPz6TdS@4JT5MOPrqDh5_H(eX1$P2SQrkvN8sTxwV>l0)Qq z0pzTuvtEAKRDkKGhhv^jk%|HQ1DdF%5oKq5BS>szk-CIke{%js?~%@$uaN3^Uz6Wf z_iyx{bZ(;9y4X&>LPV=L=d+A}7I4GkK0c1Xts{rrW1Q7apHf-))`BgC^0^F(>At1* za@e7{lq%yAkn*NH8Q1{@{lKhRg*^TfGvv!Sn*ed*x@6>M%aaqySxR|oNadYt1mpUZ z6H(rupHYf&Z z29$5g#|0MX#aR6TZ$@eGxxABRKakDYtD%5BmKp;HbG_ZbT+=81E&=XRk6m_3t9PvD zr5Cqy(v?gHcYvYvXkNH@S#Po~q(_7MOuCAB8G$a9BC##gw^5mW16cML=T=ERL7wsk zzNEayTG?mtB=x*wc@ifBCJ|irFVMOvH)AFRW8WE~U()QT=HBCe@s$dA9O!@`zAAT) zaOZ7l6vyR+Nk_OOF!ZlZmjoImKh)dxFbbR~z(cMhfeX1l7S_`;h|v3gI}n9$sSQ>+3@AFAy9=B_y$)q;Wdl|C-X|VV3w8 z2S#>|5dGA8^9%Bu&fhmVRrTX>Z7{~3V&0UpJNEl0=N32euvDGCJ>#6dUSi&PxFW*s zS`}TB>?}H(T2lxBJ!V#2taV;q%zd6fOr=SGHpoSG*4PDaiG0pdb5`jelVipkEk%FV zThLc@Hc_AL1#D&T4D=w@UezYNJ%0=f3iVRuVL5H?eeZM}4W*bomebEU@e2d`M<~uW zf#Bugwf`VezG|^Qbt6R_=U0}|=k;mIIakz99*>FrsQR{0aQRP6ko?5<7bkDN8evZ& zB@_KqQG?ErKL=1*ZM9_5?Pq%lcS4uLSzN(Mr5=t6xHLS~Ym`UgM@D&VNu8e?_=nSFtF$u@hpPSmI4Vo_t&v?>$~K4y(O~Rb*(MFy_igM7 z*~yYUyR6yQgzWnWMUgDov!!g=lInM+=lOmOk4L`O?{i&qxy&D*_qorRbDwj6?)!ef z#JLd7F6Z2I$S0iYI={rZNk*<{HtIl^mx=h>Cim*04K4+Z4IJtd*-)%6XV2(MCscPiw_a+y*?BKbTS@BZ3AUao^%Zi#PhoY9Vib4N>SE%4>=Jco0v zH_Miey{E;FkdlZSq)e<{`+S3W=*ttvD#hB8w=|2aV*D=yOV}(&p%0LbEWH$&@$X3x~CiF-?ejQ*N+-M zc8zT@3iwkdRT2t(XS`d7`tJQAjRmKAhiw{WOqpuvFp`i@Q@!KMhwKgsA}%@sw8Xo5Y=F zhRJZg)O4uqNWj?V&&vth*H#je6T}}p_<>!Dr#89q@uSjWv~JuW(>FqoJ5^ho0%K?E z9?x_Q;kmcsQ@5=}z@tdljMSt9-Z3xn$k)kEjK|qXS>EfuDmu(Z8|(W?gY6-l z@R_#M8=vxKMAoi&PwnaIYw2COJM@atcgfr=zK1bvjW?9B`-+Voe$Q+H$j!1$Tjn+* z&LY<%)L@;zhnJlB^Og6I&BOR-m?{IW;tyYC%FZ!&Z>kGjHJ6cqM-F z&19n+e1=9AH1VrVeHrIzqlC`w9=*zfmrerF?JMzO&|Mmv;!4DKc(sp+jy^Dx?(8>1 zH&yS_4yL7m&GWX~mdfgH*AB4{CKo;+egw=PrvkTaoBU+P-4u?E|&!c z)DKc;>$$B6u*Zr1SjUh2)FeuWLWHl5TH(UHWkf zLs>7px!c5n;rbe^lO@qlYLzlDVp(z?6rPZel=YB)Uv&n!2{+Mb$-vQl=xKw( zve&>xYx+jW_NJh!FV||r?;hdP*jOXYcLCp>DOtJ?2S^)DkM{{Eb zS$!L$e_o0(^}n3tA1R3-$SNvgBq;DOEo}fNc|tB%%#g4RA3{|euq)p+xd3I8^4E&m zFrD%}nvG^HUAIKe9_{tXB;tl|G<%>yk6R;8L2)KUJw4yHJXUOPM>(-+jxq4R;z8H#>rnJy*)8N+$wA$^F zN+H*3t)eFEgxLw+Nw3};4WV$qj&_D`%ADV2%r zJCPCo%{=z7;`F98(us5JnT(G@sKTZ^;2FVitXyLe-S5(hV&Ium+1pIUB(CZ#h|g)u zSLJJ<@HgrDiA-}V_6B^x1>c9B6%~847JkQ!^KLZ2skm;q*edo;UA)~?SghG8;QbHh z_6M;ouo_1rq9=x$<`Y@EA{C%6-pEV}B(1#sDoe_e1s3^Y>n#1Sw;N|}8D|s|VPd+g z-_$QhCz`vLxxrVMx3ape1xu3*wjx=yKSlM~nFgkNWb4?DDr*!?U)L_VeffF<+!j|b zZ$Wn2$TDv3C3V@BHpSgv3JUif8%hk%OsGZ=OxH@8&4`bbf$`aAMchl^qN>Eyu3JH} z9-S!x8-s4fE=lad%Pkp8hAs~u?|uRnL48O|;*DEU! zuS0{cpk%1E0nc__2%;apFsTm0bKtd&A0~S3Cj^?72-*Owk3V!ZG*PswDfS~}2<8le z5+W^`Y(&R)yVF*tU_s!XMcJS`;(Tr`J0%>p=Z&InR%D3@KEzzI+-2)HK zuoNZ&o=wUC&+*?ofPb0a(E6(<2Amd6%uSu_^-<1?hsxs~0K5^f(LsGqgEF^+0_H=uNk9S0bb!|O8d?m5gQjUKevPaO+*VfSn^2892K~%crWM8+6 z25@V?Y@J<9w%@NXh-2!}SK_(X)O4AM1-WTg>sj1{lj5@=q&dxE^9xng1_z9w9DK>| z6Iybcd0e zyi;Ew!KBRIfGPGytQ6}z}MeXCfLY0?9%RiyagSp_D1?N&c{ zyo>VbJ4Gy`@Fv+5cKgUgs~na$>BV{*em7PU3%lloy_aEovR+J7TfQKh8BJXyL6|P8un-Jnq(ghd!_HEOh$zlv2$~y3krgeH;9zC}V3f`uDtW(%mT#944DQa~^8ZI+zAUu4U(j0YcDfKR$bK#gvn_{JZ>|gZ5+)u?T$w7Q%F^;!Wk?G z(le7r!ufT*cxS}PR6hIVtXa)i`d$-_1KkyBU>qmgz-=T};uxx&sKgv48akIWQ89F{ z0XiY?WM^~;|T8zBOr zs#zuOONzH?svv*jokd5SK8wG>+yMC)LYL|vLqm^PMHcT=`}V$=nIRHe2?h)8WQa6O zPAU}d`1y(>kZiP~Gr=mtJLMu`i<2CspL|q2DqAgAD^7*$xzM`PU4^ga`ilE134XBQ z99P(LhHU@7qvl9Yzg$M`+dlS=x^(m-_3t|h>S}E0bcFMn=C|KamQ)=w2^e)35p`zY zRV8X?d;s^>Cof2SPR&nP3E+-LCkS0J$H!eh8~k0qo$}00b=7!H_I2O+Ro@3O$nPdm ztmbOO^B+IHzQ5w>@@@J4cKw5&^_w6s!s=H%&byAbUtczPQ7}wfTqxxtQNfn*u73Qw zGuWsrky_ajPx-5`R<)6xHf>C(oqGf_Fw|-U*GfS?xLML$kv;h_pZ@Kk$y0X(S+K80 z6^|z)*`5VUkawg}=z`S;VhZhxyDfrE0$(PMurAxl~<>lfZa>JZ288ULK7D` zl9|#L^JL}Y$j*j`0-K6kH#?bRmg#5L3iB4Z)%iF@SqT+Lp|{i`m%R-|ZE94Np7Pa5 zCqC^V3}B(FR340pmF*qaa}M}+h6}mqE~7Sh!9bDv9YRT|>vBNAqv09zXHMlcuhKD| zcjjA(b*XCIwJ33?CB!+;{)vX@9xns_b-VO{i0y?}{!sdXj1GM8+$#v>W7nw;+O_9B z_{4L;C6ol?(?W0<6taGEn1^uG=?Q3i29sE`RfYCaV$3DKc_;?HsL?D_fSYg}SuO5U zOB_f4^vZ_x%o`5|C@9C5+o=mFy@au{s)sKw!UgC&L35aH(sgDxRE2De%(%OT=VUdN ziVLEmdOvJ&5*tCMKRyXctCwQu_RH%;m*$YK&m;jtbdH#Ak~13T1^f89tn`A%QEHWs~jnY~E}p_Z$XC z=?YXLCkzVSK+Id`xZYTegb@W8_baLt-Fq`Tv|=)JPbFsKRm)4UW;yT+J`<)%#ue9DPOkje)YF2fsCilK9MIIK>p*`fkoD5nGfmLwt)!KOT+> zOFq*VZktDDyM3P5UOg`~XL#cbzC}eL%qMB=Q5$d89MKuN#$6|4gx_Jt0Gfn8w&q}%lq4QU%6#jT*MRT% zrLz~C8FYKHawn-EQWN1B75O&quS+Z81(zN)G>~vN8VwC+e+y(`>HcxC{MrJ;H1Z4k zZWuv$w_F0-Ub%MVcpIc){4PGL^I7M{>;hS?;eH!;gmcOE66z3;Z1Phqo(t zVP(Hg6q#0gIKgsg7L7WE!{Y#1nI(45tx2{$34dDd#!Z0NIyrm)HOn5W#7;f4pQci# zDW!FI(g4e668kI9{2+mLwB+=#9bfqgX%!B34V-$wwSN(_cm*^{y0jQtv*4}eO^sOV z*9xoNvX)c9isB}Tgx&ZRjp3kwhTVK?r9;n!x>^XYT z@Q^7zp{rkIs{2mUSE^2!Gf6$6;j~&4=-0cSJJDizZp6LTe8b45;{AKM%v99}{{FfC zz709%u0mC=1KXTo(=TqmZQ;c?$M3z(!xah>aywrj40sc2y3rKFw4jCq+Y+u=CH@_V zxz|qeTwa>+<|H%8Dz5u>ZI5MmjTFwXS-Fv!TDd*`>3{krWoNVx$<133`(ftS?ZPyY z&4@ah^3^i`vL$BZa>O|Nt?ucewzsF)0zX3qmM^|waXr=T0pfIb0*$AwU=?Ipl|1Y; z*Pk6{C-p4MY;j@IJ|DW>QHZQJcp;Z~?8(Q+Kk3^0qJ}SCk^*n4W zu9ZFwLHUx-$6xvaQ)SUQcYd6fF8&x)V`1bIuX@>{mE$b|Yd(qomn3;bPwnDUc0F=; zh*6_((%bqAYQWQ~odER?h>1mkL4kpb3s7`0m@rDKGU*oyF)$j~Ffd4fXV$?`f~rHf zB%Y)@5SXZvfwm10RY5X?TEo)PK_`L6qgBp=#>fO49$D zDq8Ozj0q6213tV5Qq=;fZ0$|KroY{Dz=l@lU^J)?Ko@ti20TRplXzphBi>XGx4bou zEWrkNjz0t5j!_ke{g5I#PUlEU$Km8g8TE|XK=MkU@PT4T><2OVamoK;wJ}3X0L$vX zgd7gNa359*nc)R-0!`2X@FOTB`+oETOPc=ubp5R)VQgY+5BTZZJ2?9QwnO=dnulIUF3gFn;BODC2)65)HeVd%t86sL7Rv^Y+nbn+&l z6BAJY(ETvwI)Ts$aiE8rht4KD*qNyE{8{x6R|%akbTBzw;2+6Echkt+W+`u^XX z_z&x%n@vG-+vuvg^_??!{yS%8zW-#zn-LkA z5&1^$^{lnmUON?}LBF8_K|(?T0Ra(xUH{($5eN!MR#ZihR#HxkUPe+_R8Cn`RRs(P z_^*#_XlXmGv7!4;*Y%p4nw?{bNp@UZHv1?Um8r6)Fei3p@ClJn0ECfg1hkeuUU@Or zDaPa;U3fE=3L}DooL;8f;P0ipPt0Z~9P0)lbStMS)ag54=uL9ia-Lm3nh|@(Y?B`; zx_#arJIpXH!U{fbCbI^17}6Ri*H<>OLR%c|^mh8+)*h~K8Z!9)DPf zR2h?lbDZQ`p9P;&DQ4F0sur@TMa!Y}S8irn(%d-gi0*WxxCSk*A?3lGh=gcYN?FGl z7D=Js!i~0=u3rox^eO3i@$0=n{K1lPNU zwmfjRVmLOCRfe=seV&P*1Iq=^i`502keY8Uy-WNPwVNNtJFx?IwAyRPZo2Wo1+S(xF37LJZ~%i)kpFQ3Fw=mXfd@>%+)RpYQLnr}B~~zoof(JVm^^&f zxKV^+3D3$A1G;qh4gPVjhrC8e(VYUHv#dy^)(RoUFM?o%W-EHxufuWf(l*@-l+7vt z=l`qmR56K~F|v<^Pd*p~1_y^P0P^aPC##d8+HqX4IR1gu+7w#~TBFphJxF)T$2WEa zxa?H&6=Qe7d(#tha?_1uQys2KtHQ{)Qco)qwGjrdNL7thd^G5i8Os)CHqc>iOidS} z%nFEDdm=GXBw=yXe1W-ShHHFb?Cc70+$W~z_+}nAoHFYI1MV1wZegw*0y^tC*s%3h zhD3tN8b=Gv&rj}!SUM6|ajSPp*58KR7MPpI{oAJCtY~JECm)*m_x>AZEu>DFgUcby z1Qaw8lU4jZpQ_$;*7RME+gq1KySGG#Wql>aL~k9tLrSO()LWn*q&YxHEuzmwd1?aAtI zBJ>P=&$=l1efe1CDU;`Fd+_;&wI07?V0aAIgc(!{a z0Jg6Y=inXc3^n!U0Atk`iCFIQooHqcWhO(qrieUOW8X(x?(RD}iYDLMjSwffH2~tB z)oDgNBLB^AJBM1M^c5HdRx6fBfka`(LD-qrlh5jqH~);#nw|iyp)()xVYak3;Ybik z0j`(+69aK*B>)e_p%=wu8XC&9e{AO4c~O1U`5X9}?0mrd*m$_EUek{R?DNSh(=br# z#Q61gBzEpmy`$pA*6!87 zSDD+=@fTY7<4A?GLqpA?Pb2z$pbCc4B4zL{BeZ?F-8`s$?>*lXXtn*NC61>|*w7J* z$?!iB{6R-0=KFmyp1nnEmLsA-H0a6l+1uaH^g%c(p{iT&YFrbQ$&PRb8Up#X3@Zsk zD^^&LK~111%cqlP%!_gFNa^dTYT?rhkGl}5=fL{a`UViaXWI$k-UcHJwmaH1s=S$4 z%4)PdWJX;hh5UoK?6aWoyLxX&NhNRqKam7tcOkLh{%j3K^4Mgx1@i|Pi&}<^5>hs5 zm8?uOS>%)NzT(%PjVPGa?X%`N2TQCKbeH2l;cTnHiHppPSJ<7y-yEIiC!P*ikl&!B z%+?>VttCOQM@ShFguHVjxX^?mHX^hSaO_;pnyh^v9EumqSZTi+#f&_Vaija0Q-e*| z7ulQj6Fs*bbmsWp{`auM04gGwsYYdNNZcg|ph0OgD>7O}Asn7^Z=eI>`$2*v78;sj-}oMoEj&@)9+ycEOo92xSyY344^ z11Hb8^kdOvbf^GNAK++bYioknrpdN>+u8R?JxG=!2Kd9r=YWCOJYXYuM0cOq^FhEd zBg2puKy__7VT3-r*dG4c62Wgxi52EMCQ`bKgf*#*ou(D4-ZN$+mg&7$u!! z-^+Z%;-3IDwqZ|K=ah85OLwkO zKxNBh+4QHh)u9D?MFtpbl)us}9+V!D%w9jfAMYEb>%$A;u)rrI zuBudh;5PN}_6J_}l55P3l_)&RMlH{m!)ai-i$g)&*M`eN$XQMw{v^r@-125^RRCF0 z^2>|DxhQw(mtNEI2Kj(;KblC7x=JlK$@78`O~>V!`|1Lm-^JR$-5pUANAnb(5}B}JGjBsliK4& zk6y(;$e&h)lh2)L=bvZKbvh@>vLlreBdH8No2>$#%_Wp1U0N7Ank!6$dFSi#xzh|( zRi{Uw%-4W!{IXZ)fWx@XX6;&(m_F%c6~X8hx=BN1&q}*( zoaNjWabE{oUPb!Bt$eyd#$5j9rItB-h*5JiNi(v^e|XKAj*8(k<5-2$&ZBR5fF|JA z9&m4fbzNQnAU}r8ab>fFV%J0z5awe#UZ|bz?Ur)U9bCIKWEzi2%A+5CLqh?}K4JHi z4vtM;+uPsVz{Lfr;78W78gC;z*yTch~4YkLr&m-7%-xc ztw6Mh2d>_iO*$Rd8(-Cr1_V8EO1f*^@wRoSozS) zy1UoC@pruAaC8Z_7~_w4Q6n*&B0AjOmMWa;sIav&gu z|J5&|{=a@vR!~k-OjKEgPFCzcJ>#A1uL&7xTDn;{XBdeM}V=l3B8fE1--DHjSaxoSjNKEM9|U9#m2<3>n{Iuo`r3UZp;>GkT2YBNAh|b z^jTq-hJp(ebZh#Lk8hVBP%qXwv-@vbvoREX$TqRGTgEi$%_F9tZES@z8Bx}$#5eeG zk^UsLBH{bc2VBW)*EdS({yw=?qmevwi?BL6*=12k9zM5gJv1>y#ML4!)iiPzVaH9% zgSImetD@dam~e>{LvVh!phhzpW+iFvWpGT#CVE5TQ40n%F|p(sP5mXxna+Ev7PDwA zamaV4m*^~*xV+&p;W749xhb_X=$|LD;FHuB&JL5?*Y2-oIT(wYY2;73<^#46S~Gx| z^cez%V7x$81}UWqS13Gz80379Rj;6~WdiXWOSsdmzY39L;Hg3MH43o*y8ibNBBH`(av4|u;YPq%{R;IuYow<+GEsf@R?=@tT@!}?#>zIIn0CoyV!hq3mw zHj>OOjfJM3F{RG#6ujzo?y32m^tgSXf@v=J$ELdJ+=5j|=F-~hP$G&}tDZsZE?5rX ztGj`!S>)CFmdkccxM9eGIcGnS2AfK#gXwj%esuIBNJQP1WV~b~+D7PJTmWGTSDrR` zEAu4B8l>NPuhsk5a`rReSya2nfV1EK01+G!x8aBdTs3Io$u5!6n6KX%uv@DxAp3F@{4UYg4SWJtQ-W~0MDb|j-$lwVn znAm*Pl!?Ps&3wO=R115RWKb*JKoexo*)uhhHBncEDMSVa_PyA>k{Zm2(wMQ(5NM3# z)jkza|GoWEQo4^s*wE(gHz?Xsg4`}HUAcs42cM1-qq_=+=!Gk^y710j=66(cSWqUe zklbm8+zB_syQv5A2rj!Vbw8;|$@C!vfNmNV!yJIWDQ>{+2x zKjuFX`~~HKG~^6h5FntRpnnHt=D&rq0>IJ9#F0eM)Y-)GpRjiN7gkA8wvnG#K=q{q z9dBn8_~wm4J<3J_vl|9H{7q6u2A!cW{bp#r*-f{gOV^e=8S{nc1DxMHFwuM$;aVI^ zz6A*}m8N-&x8;aunp1w7_vtB*pa+OYBw=TMc6QK=mbA-|Cf* zvyh8D4LRJImooUaSb7t*fVfih<97Gf@VE0|z>NcBwBQze);Rh!k3K_sfunToZY;f2 z^HmC4KjHRVg+eKYj;PRN^|E0>Gj_zagfRbrki68I^#~6-HaHg3BUW%+clM1xQEdPYt_g<2K+z!$>*$9nQ>; zf9Bei{?zY^-e{q_*|W#2rJG`2fy@{%6u0i_VEWTq$*(ZN37|8lFFFt)nCG({r!q#9 z5VK_kkSJ3?zOH)OezMT{!YkCuSSn!K#-Rhl$uUM(bq*jY? zi1xbMVthJ`E>d>(f3)~fozjg^@eheMF6<)I`oeJYx4*+M&%c9VArn(OM-wp%M<-`x z7sLP1&3^%Nld9Dhm@$3f2}87!quhI@nwd@3~fZl_3LYW-B?Ia>ui`ELg z&Qfe!7m6ze=mZ`Ia9$z|ARSw|IdMpooY4YiPN8K z4B(ts3p%2i(Td=tgEHX z0UQ_>URBtG+-?0E;E7Ld^dyZ;jjw0}XZ(}-QzC6+NN=40oDb2^v!L1g9xRvE#@IBR zO!b-2N7wVfLV;mhEaXQ9XAU+>=XVA6f&T4Z-@AX!leJ8obP^P^wP0aICND?~w&NykJ#54x3_@r7IDMdRNy4Hh;h*!u(Ol(#0bJdwEo$5437-UBjQ+j=Ic>Q2z` zJNDf0yO6@mr6y1#n3)s(W|$iE_i8r@Gd@!DWDqZ7J&~gAm1#~maIGJ1sls^gxL9LLG_NhU!pTGty!TbhzQnu)I*S^54U6Yu%ZeCg`R>Q zhBv$n5j0v%O_j{QYWG!R9W?5_b&67KB$t}&e2LdMvd(PxN6Ir!H4>PNlerpBL>Zvyy!yw z-SOo8caEpDt(}|gKPBd$qND5#a5nju^O>V&;f890?yEOfkSG^HQVmEbM3Ugzu+UtH zC(INPDdraBN?P%kE;*Ae%Wto&sgw(crfZ#Qy(<4nk;S|hD3j{IQRI6Yq|f^basLY; z-HB&Je%Gg}Jt@={_C{L$!RM;$$|iD6vu#3w?v?*;&()uB|I-XqEKqZPS!reW9JkLewLb!70T7n`i!gNtb1%vN- zySZj{8-1>6E%H&=V}LM#xmt`J3XQoaD|@XygXjdZ1+P77-=;=eYpoEQ01B@L*a(uW zrZeZz?HJsw_4g0vhUgkg@VF8<-X$B8pOqCuWAl28uB|@r`19DTUQQsb^pfqB6QtiT z*`_UZ`fT}vtUY#%sq2{rchyfu*pCg;uec2$-$N_xgjZcoumE5vSI{+s@iLWoz^Mf; zuI8kDP{!XY6OP~q5}%1&L}CtfH^N<3o4L@J@zg1-mt{9L`s^z$Vgb|mr{@WiwAqKg zp#t-lhrU>F8o0s1q_9y`gQNf~Vb!F%70f}$>i7o4ho$`uciNf=xgJ>&!gSt0g;M>*x4-`U)ysFW&Vs^Vk6m%?iuWU+o&m(2Jm26Y(3%TL; zA7T)BP{WS!&xmxNw%J=$MPfn(9*^*TV;$JwRy8Zl*yUZi8jWYF>==j~&S|Xinsb%c z2?B+kpet*muEW7@AzjBA^wAJBY8i|#C{WtO_or&Nj2{=6JTTX05}|H>N2B|Wf!*3_ z7hW*j6p3TvpghEc6-wufFiY!%-GvOx*bZrhZu+7?iSrZL5q9}igiF^*R3%DE4aCHZ zqu>xS8LkW+Auv%z-<1Xs92u23R$nk@Pk}MU5!gT|c7vGlEA%G^2th&Q*zfg%-D^=f z&J_}jskj|Q;73NP4<4k*Y%pXPU2Thoqr+5uH1yEYM|VtBPW6lXaetokD0u z9qVek6Q&wk)tFbQ8(^HGf3Wp16gKmr>G;#G(HRBx?F`9AIRboK+;OfHaLJ(P>IP0w zyTbTkx_THEOs%Q&aPrxbZrJlio+hCC_HK<4%f3ZoSAyG7Dn`=X=&h@m*|UYO-4Hq0 z-Bq&+Ie!S##4A6OGoC~>ZW`Y5J)*ouaFl_e9GA*VSL!O_@xGiBw!AF}1{tB)z(w%c zS1Hmrb9OC8>0a_$BzeiN?rkPLc9%&;1CZW*4}CDDNr2gcl_3z+WC15&H1Zc2{o~i) z)LLW=WQ{?ricmC`G1GfJ0Yp4Dy~Ba;j6ZV4r{8xRs`13{dD!xXmr^Aga|C=iSmor% z8hi|pTXH)5Yf&v~exp3o+sY4B^^b*eYkkCYl*T{*=-0HniSA_1F53eCb{x~1k3*`W zr~};p1A`k{1DV9=UPnLDgz{aJH=-LQo<5%+Em!DNN252xwIf*wF_zS^!(XSm(9eoj z=*dXG&n0>)_)N5oc6v!>-bd(2ragD8O=M|wGW z!xJQS<)u70m&6OmrF0WSsr@I%T*c#Qo#Ha4d3COcX+9}hM5!7JIGF>7<~C(Ear^Sn zm^ZFkV6~Ula6+8S?oOROOA6$C&q&dp`>oR-2Ym3(HT@O7Sd5c~+kjrmM)YmgPH*tL zX+znN>`tv;5eOfX?h{AuX^LK~V#gPCu=)Tigtq9&?7Xh$qN|%A$?V*v=&-2F$zTUv z`C#WyIrChS5|Kgm_GeudCFf;)!WH7FI60j^0o#65o6`w*S7R@)88n$1nrgU(oU0M9 zx+EuMkC>(4j1;m6NoGqEkpJYJ?vc|B zOlwT3t&UgL!pX_P*6g36`ZXQ; z9~Cv}ANFnJGp(;ZhS(@FT;3e)0)Kp;h^x;$*xZn*k0U6-&FwI=uOGaODdrsp-!K$Ac32^c{+FhI-HkYd5v=`PGsg%6I`4d9Jy)uW0y%) zm&j^9WBAp*P8#kGJUhB!L?a%h$hJgQrx!6KCB_TRo%9{t0J7KW8!o1B!NC)VGLM5! zpZy5Jc{`r{1e(jd%jsG7k%I+m#CGS*BPA65ZVW~fLYw0dA-H_}O zrkGFL&P1PG9p2(%QiEWm6x;U-U&I#;Em$nx-_I^wtgw3xUPVVu zqSuKnx&dIT-XT+T10p;yjo1Y)z(x1fb8Dzfn8e yu?e%!_ptzGB|8GrCfu%p?(_ zQccdaaVK$5bz;*rnyK{_SQYM>;aES6Qs^lj9lEs6_J+%nIiuQC*fN;z8md>r_~Mfl zU%p5Dt_YT>gQqfr@`cR!$NWr~+`CZb%dn;WtzrAOI>P_JtsB76PYe*<%H(y>qx-`Kq!X_; z<{RpAqYhE=L1r*M)gNF3B8r(<%8mo*SR2hu zccLRZwGARt)Hlo1euqTyM>^!HK*!Q2P;4UYrysje@;(<|$&%vQekbn|0Ruu_Io(w4#%p6ld2Yp7tlA`Y$cciThP zKzNGIMPXX%&Ud0uQh!uQZz|FB`4KGD?3!ND?wQt6!n*f4EmCoJUh&b?;B{|lxs#F- z31~HQ`SF4x$&v00@(P+j1pAaj5!s`)b2RDBp*PB=2IB>oBF!*6vwr7Dp%zpAx*dPr zb@Zjq^XjN?O4QcZ*O+8>)|HlrR>oD*?WQl5ri3R#2?*W6iJ>>kH%KnnME&TT@ZzrHS$Q%LC?n|e>V+D+8D zYc4)QddFz7I8#}y#Wj6>4P%34dZH~OUDb?uP%-E zwjXM(?Sg~1!|wI(RVuxbu)-rH+O=igSho_pDCw(c6b=P zKk4ATlB?bj9+HHlh<_!&z0rx13K3ZrAR8W)!@Y}o`?a*JJsD+twZIv`W)@Y?Amu_u zz``@-e2X}27$i(2=9rvIu5uTUOVhzwu%mNazS|lZb&PT;XE2|B&W1>=B58#*!~D&) zfVmJGg8UdP*fx(>Cj^?yS^zH#o-$Q-*$SnK(ZVFkw+er=>N^7!)FtP3y~Xxnu^nzY zikgB>Nj0%;WOltWIob|}%lo?_C7<``a5hEkx&1ku$|)i>Rh6@3h*`slY=9U}(Ql_< zaNG*J8vb&@zpdhAvv`?{=zDedJ23TD&Zg__snRAH4eh~^oawdYi6A3w8<Ozh@Kw)#bdktM^GVb zrG08?0bG?|NG+w^&JvD*7LAbjED{_Zkc`3H!My>0u5Q}m!+6VokMLXxl`Mkd=g&Xx z-a>m*#G3SLlhbKB!)tnzfWOBV;u;ftU}S!NdD5+YtOjLg?X}dl>7m^gOpihrf1;PY zvll&>dIuUGs{Qnd- zwIR3oIrct8Va^Tm0t#(bJD7c$Z7DO9*7NnRZorrSm`b`cxz>OIC;jSE3DO8`hX955ui`s%||YQtt2 z5DNA&pG-V+4oI2s*x^>-$6J?p=I>C|9wZF8z;VjR??Icg?1w2v5Me+FgAeGGa8(3S z4vg*$>zC-WIVZtJ7}o9{D-7d>zCe|z#<9>CFve-OPAYsneTb^JH!Enaza#j}^mXy1 z+ULn^10+rWLF6j2>Ya@@Kq?26>AqK{A_| zQKb*~F1>sE*=d?A?W7N2j?L09_7n+HGi{VY;MoTGr_)G9)ot$p!-UY5zZ2Xtbm=t z@dpPSGwgH=QtIcEulQNI>S-#ifbnO5EWkI;$A|pxJd885oM+ zGZ0_0gDvG8q2xebj+fbCHYfAXuZStH2j~|d^sBAzo46(K8n59+T6rzBwK)^rfPT+B zyIFw)9YC-V^rhtK`!3jrhmW-sTmM+tPH+;nwjL#-SjQPUZ53L@A>y*rt(#M(qsiB2 zx6B)dI}6Wlsw%bJ8h|(lhkJVogQZA&n{?Vgs6gNSXzuZpEyu*xySy8ro07QZ7Vk1!3tJphN_5V7qOiyK8p z#@jcDD8nmtYi1^l8ml;AF<#IPK?!pqf9D4moYk>d99Im}Jtwj6c#+A;f)CQ*f-hZ< z=p_T86jog%!p)D&5g9taSwYi&eP z#JuEK%+NULWus;0w32-SYFku#i}d~+{Pkho&^{;RxzP&0!RCm3-9K6`>KZpnzS6?L z^H^V*s!8<>x8bomvD%rh>Zp3>Db%kyin;qtl+jAv8Oo~1g~mqGAC&Qi_wy|xEt2iz zWAJEfTV%cl2Cs<1L&DLRVVH05EDq`pH7Oh7sR`NNkL%wi}8n>IXcO40hp+J+sC!W?!krJf!GJNE8uj zg-y~Ns-<~D?yqbzVRB}G>0A^f0!^N7l=$m0OdZuqAOQqLc zX?AEGr1Ht+inZ-Qiwnl@Z0qukd__a!C*CKuGdy5#nD7VUBM^6OCpxCa2A(X;e0&V4 zM&WR8+wErQ7UIc6LY~Q9x%Sn*Tn>>P`^t&idaOEnOd(Ufw#>NoR^1QdhJ8s`h^|R_ zXX`c5*O~Xdvh%q;7L!_!ohf$NfEBmCde|#uVZvEo>OfEq%+Ns7&_f$OR9xsihRpBb z+cjk8LyDm@U{YN>+r46?nn{7Gh(;WhFw6GAxtcKD+YWV?uge>;+q#Xx4!GpRkVZYu zzsF}1)7$?%s9g9CH=Zs+B%M_)+~*j3L0&Q9u7!|+T`^O{xE6qvAP?XWv9_MrZKdo& z%IyU)$Q95AB4!#hT!_dA>4e@zjOBD*Y=XjtMm)V|+IXzjuM;(l+8aA5#Kaz_$rR6! zj>#&^DidYD$nUY(D$mH`9eb|dtV0b{S>H6FBfq>t5`;OxA4Nn{J(+XihF(stSche7$es&~N$epi&PDM_N`As;*9D^L==2Q7Z2zD+CiU(|+-kL*VG+&9!Yb3LgPy?A zm7Z&^qRG_JIxK7-FBzZI3Q<;{`DIxtc48k> zc|0dmX;Z=W$+)qE)~`yn6MdoJ4co;%!`ddy+FV538Y)j(vg}5*k(WK)KWZ3WaOG!8 z!syGn=s{H$odtpqFrT#JGM*utN7B((abXnpDM6w56nhw}OY}0TiTG1#f*VFZr+^-g zbP10`$LPq_;PvrA1XXlyx2uM^mrjTzX}w{yuLo-cOClE8MMk47T25G8M!9Z5ypOSV zAJUBGEg5L2fY)ZGJb^E34R2zJ?}Vf>{~gB!8=5Z) z9y$>5c)=;o0HeHHSuE4U)#vG&KF|I%-cF6f$~pdYJWk_dD}iOA>iA$O$+4%@>JU08 zS`ep)$XLPJ+n0_i@PkF#ri6T8?ZeAot$6JIYHm&P6EB=BiaNY|aA$W0I+nz*zkz_z zkEru!tj!QUffq%)8y0y`T&`fuus-1p>=^hnBiBqD^hXrPs`PY9tU3m0np~rISY09> z`P3s=-kt_cYcxWd{de@}TwSqg*xVhp;E9zCsnXo6z z?f&Sv^U7n4`xr=mXle94HzOdN!2kB~4=%)u&N!+2;z6UYKUDqi-s6AZ!haB;@&B`? z_TRX0%@suz^TRdCb?!vNJYPY8L_}&07uySH9%W^Tc&1pia6y1q#?*Drf}GjGbPjBS zbOPcUY#*$3sL2x4v_i*Y=N7E$mR}J%|GUI(>WEr+28+V z%v5{#e!UF*6~G&%;l*q*$V?&r$Pp^sE^i-0$+RH3ERUUdQ0>rAq2(2QAbG}$y{de( z>{qD~GGuOk559Y@%$?N^1ApVL_a704>8OD%8Y%8B;FCt%AoPu8*D1 zLB5X>b}Syz81pn;xnB}%0FnwazlWfUV)Z-~rZg6~b z6!9J$EcE&sEbzcy?CI~=boWA&eeIa%z(7SE^qgVLz??1Vbc1*aRvc%Mri)AJaAG!p z$X!_9Ds;Zz)f+;%s&dRcJt2==P{^j3bf0M=nJd&xwUGlUFn?H=2W(*2I2Gdu zv!gYCwM10aeus)`RIZSrCK=&oKaO_Ry~D1B5!y0R=%!i2*KfXGYX&gNv_u+n9wiR5 z*e$Zjju&ODRW3phN925%S(jL+bCHv6rZtc?!*`1TyYXT6%Ju=|X;6D@lq$8T zW{Y|e39ioPez(pBH%k)HzFITXHvnD6hw^lIoUMA;qAJ^CU?top1fo@s7xT13Fvn1H z6JWa-6+FJF#x>~+A;D~;VDs26>^oH0EI`IYT2iagy23?nyJ==i{g4%HrAf1-*v zK1)~@&(KkwR7TL}L(A@C_S0G;-GMDy=MJn2$FP5s<%wC)4jC5PXoxrQBFZ_k0P{{s@sz+gX`-!=T8rcB(=7vW}^K6oLWMmp(rwDh}b zwaGGd>yEy6fHv%jM$yJXo5oMAQ>c9j`**}F?MCry;T@47@r?&sKHgVe$MCqk#Z_3S z1GZI~nOEN*P~+UaFGnj{{Jo@16`(qVNtbU>O0Hf57-P>x8Jikp=`s8xWs^dAJ9lCQ z)GFm+=OV%AMVqVATtN@|vp61VVAHRn87}%PC^RAzJ%JngmZTasWBAWsoAqBU+8L8u z4A&Pe?fmTm0?mK-BL9t+{y7o(7jm+RpOhL9KnY#E&qu^}B6=K_dB}*VlSEiC9fn)+V=J;OnN)Ta5v66ic1rG+dGAJ1 z1%Zb_+!$=tQ~lxQrzv3x#CPb?CekEkA}0MYSgx$Jdd}q8+R=ma$|&1a#)TQ=l$1tQ z=tL9&_^vJ)Pk}EDO-va`UCT1m#Uty1{v^A3P~83_#v^ozH}6*9mIjIr;t3Uv%@VeW zGL6(CwCUp)Jq%G0bIG%?{_*Y#5IHf*5M@wPo6A{$Um++Co$wLC=J1aoG93&T7Ho}P z=mGEPP7GbvoG!uD$k(H3A$Z))+i{Hy?QHdk>3xSBXR0j!11O^mEe9RHmw!pvzv?Ua~2_l2Yh~_!s1qS`|0~0)YsbHSz8!mG)WiJE| z2f($6TQtt6L_f~ApQYQKSb=`053LgrQq7G@98#igV>y#i==-nEjQ!XNu9 z~;mE+gtj4IDDNQJ~JVk5Ux6&LCSFL!y=>79kE9=V}J7tD==Ga+IW zX)r7>VZ9dY=V&}DR))xUoV!u(Z|%3ciQi_2jl}3=$Agc(`RPb z8kEBpvY>1FGQ9W$n>Cq=DIpski};nE)`p3IUw1Oz0|wxll^)4dq3;CCY@RyJgFgc# zKouFh!`?Xuo{IMz^xi-h=StCis_M7yq$u) z?XHvw*HP0VgR+KR6wI)jEMX|ssqYvSf*_3W8zVTQzD?3>H!#>InzpSO)@SC8q*ii- z%%h}_#0{4JG;Jm`4zg};BPTGkYamx$Xo#O~lBirRY)q=5M45n{GCfV7h9qwyu1NxOMoP4)jjZMxmT|IQQh0U7C$EbnMN<3)Kk?fFHYq$d|ICu>KbY_hO zTZM+uKHe(cIZfEqyzyYSUBZa8;Fcut-GN!HSA9ius`ltNebF46ZX_BbZNU}}ZOm{M2&nANL9@0qvih15(|`S~z}m&h!u4x~(%MAO$jHRWNfuxWF#B)E&g3ghSQ9|> z(MFaLQj)NE0lowyjvg8z0#m6FIuKE9lDO~Glg}nSb7`~^&#(Lw{}GVOS>U)m8bF}x zVjbXljBm34Cs-yM6TVusr+3kYFjr28STT3g056y3cH5Tmge~ASxBj z%|yb>$eF;WgrcOZf569sDZOVwoo%8>XO>XQOX1OyN9I-SQgrm;U;+#3OI(zrWyow3 zk==|{lt2xrQ%FIXOTejR>;wv(Pb8u8}BUpx?yd(Abh6? zsoO3VYWkeLnF43&@*#MQ9-i-d0t*xN-UEyNKeyNMHw|A(k(_6QKO=nKMCxD(W(Yop zsRQ)QeL4X3Lxp^L%wzi2-WVSsf61dqliPUM7srDB?Wm6Lzn0&{*}|IsKQW;02(Y&| zaTKv|`U(pSzuvR6Rduu$wzK_W-Y-7>7s?G$)U}&uK;<>vU}^^ns@Z!p+9?St1s)dG zK%y6xkPyyS1$~&6v{kl?Md6gwM|>mt6Upm>oa8RLD^8T{0?HC!Z>;(Bob7el(DV6x zi`I)$&E&ngwFS@bi4^xFLAn`=fzTC;aimE^!cMI2n@Vo%Ae-ne`RF((&5y6xsjjAZ zVguVoQ?Z9uk$2ON;ersE%PU*xGO@T*;j1BO5#TuZKEf(mB7|g7pcEA=nYJ{s3vlbg zd4-DUlD{*6o%Gc^N!Nptgay>j6E5;3psI+C3Q!1ZIbeCubW%w4pq9)MSDyB{HLm|k zxv-{$$A*pS@csolri$Ge<4VZ}e~78JOL-EVyrbxKra^d{?|NnPp86!q>t<&IP07?Z z^>~IK^k#OEKgRH+LjllZXk7iA>2cfH6+(e&9ku5poo~6y{GC5>(bRK7hwjiurqAiZ zg*DmtgY}v83IjE&AbiWgMyFbaRUPZ{lYiz$U^&Zt2YjG<%m((&_JUbZcfJ22(>bi5 z!J?<7AySj0JZ&<-qXX;mcV!f~>G=sB0KnjWca4}vrtunD^1TrpfeS^4dvFr!65knK zZh`d;*VOkPs4*-9kL>$GP0`(M!j~B;#x?Ba~&s6CopvO86oM?-? zOw#dIRc;6A6T?B`Qp%^<U5 z19x(ywSH$_N+Io!6;e?`tWaM$`=Db!gzx|lQ${DG!zb1Zl&|{kX0y6xvO1o z220r<-oaS^^R2pEyY;=Qllqpmue|5yI~D|iI!IGt@iod{Opz@*ml^w2bNs)p`M(Io z|E;;m*Xpjd9l)4G#KaWfV(t8YUn@A;nK^#xgv=LtnArX|vWQVuw3}B${h+frU2>9^ z!l6)!Uo4`5k`<<;E(ido7M6lKTgWezNLq>U*=uz&s=cc$1%>VrAeOoUtA|T6gO4>UNqsdK=NF*8|~*sl&wI=x9-EGiq*aqV!(VVXA57 zw9*o6Ir8Lj1npUXvlevtn(_+^X5rzdR>#(}4YcB9O50q97%rW2me5_L=%ffYPUSRc z!vv?Kv>dH994Qi>U(a<0KF6NH5b16enCp+mw^Hb3Xs1^tThFpz!3QuN#}KBbww`(h z7GO)1olDqy6?T$()R7y%NYx*B0k_2IBiZ14&8|JPFxeMF{vSTxF-Vi3+ZOI=Thq2} zyQgjYY1_7^ZQHh{?P))4+qUiQJLi1&{yE>h?~jU%tjdV0h|FENbM3X(KnJdPKc?~k zh=^Ixv*+smUll!DTWH!jrV*wSh*(mx0o6}1@JExzF(#9FXgmTXVoU+>kDe68N)dkQ zH#_98Zv$}lQwjKL@yBd;U(UD0UCl322=pav<=6g>03{O_3oKTq;9bLFX1ia*lw;#K zOiYDcBJf)82->83N_Y(J7Kr_3lE)hAu;)Q(nUVydv+l+nQ$?|%MWTy`t>{havFSQloHwiIkGK9YZ79^9?AZo0ZyQlVR#}lF%dn5n%xYksXf8gnBm=wO7g_^! zauQ-bH1Dc@3ItZ-9D_*pH}p!IG7j8A_o94#~>$LR|TFq zZ-b00*nuw|-5C2lJDCw&8p5N~Z1J&TrcyErds&!l3$eSz%`(*izc;-?HAFD9AHb-| z>)id`QCrzRws^9(#&=pIx9OEf2rmlob8sK&xPCWS+nD~qzU|qG6KwA{zbikcfQrdH z+ zQg>O<`K4L8rN7`GJB0*3<3`z({lWe#K!4AZLsI{%z#ja^OpfjU{!{)x0ZH~RB0W5X zTwN^w=|nA!4PEU2=LR05x~}|B&ZP?#pNgDMwD*ajI6oJqv!L81gu=KpqH22avXf0w zX3HjbCI!n9>l046)5rr5&v5ja!xkKK42zmqHzPx$9Nn_MZk`gLeSLgC=LFf;H1O#B zn=8|^1iRrujHfbgA+8i<9jaXc;CQBAmQvMGQPhFec2H1knCK2x!T`e6soyrqCamX% zTQ4dX_E*8so)E*TB$*io{$c6X)~{aWfaqdTh=xEeGvOAN9H&-t5tEE-qso<+C!2>+ zskX51H-H}#X{A75wqFe-J{?o8Bx|>fTBtl&tcbdR|132Ztqu5X0i-pisB-z8n71%q%>EF}yy5?z=Ve`}hVh{Drv1YWL zW=%ug_&chF11gDv3D6B)Tz5g54H0mDHNjuKZ+)CKFk4Z|$RD zfRuKLW`1B>B?*RUfVd0+u8h3r-{@fZ{k)c!93t1b0+Q9vOaRnEn1*IL>5Z4E4dZ!7 ztp4GP-^1d>8~LMeb}bW!(aAnB1tM_*la=Xx)q(I0Y@__Zd$!KYb8T2VBRw%e$iSdZ zkwdMwd}eV9q*;YvrBFTv1>1+}{H!JK2M*C|TNe$ZSA>UHKk);wz$(F$rXVc|sI^lD zV^?_J!3cLM;GJuBMbftbaRUs$;F}HDEDtIeHQ)^EJJ1F9FKJTGH<(Jj`phE6OuvE) zqK^K`;3S{Y#1M@8yRQwH`?kHMq4tHX#rJ>5lY3DM#o@or4&^_xtBC(|JpGTfrbGkA z2Tu+AyT^pHannww!4^!$5?@5v`LYy~T`qs7SYt$JgrY(w%C+IWA;ZkwEF)u5sDvOK zGk;G>Mh&elvXDcV69J_h02l&O;!{$({fng9Rlc3ID#tmB^FIG^w{HLUpF+iB`|
NnX)EH+Nua)3Y(c z&{(nX_ht=QbJ%DzAya}!&uNu!4V0xI)QE$SY__m)SAKcN0P(&JcoK*Lxr@P zY&P=}&B3*UWNlc|&$Oh{BEqwK2+N2U$4WB7Fd|aIal`FGANUa9E-O)!gV`((ZGCc$ zBJA|FFrlg~9OBp#f7aHodCe{6= zay$6vN~zj1ddMZ9gQ4p32(7wD?(dE>KA2;SOzXRmPBiBc6g`eOsy+pVcHu=;Yd8@{ zSGgXf@%sKKQz~;!J;|2fC@emm#^_rnO0esEn^QxXgJYd`#FPWOUU5b;9eMAF zZhfiZb|gk8aJIw*YLp4!*(=3l8Cp{(%p?ho22*vN9+5NLV0TTazNY$B5L6UKUrd$n zjbX%#m7&F#U?QNOBXkiiWB*_tk+H?N3`vg;1F-I+83{M2!8<^nydGr5XX}tC!10&e z7D36bLaB56WrjL&HiiMVtpff|K%|*{t*ltt^5ood{FOG0<>k&1h95qPio)2`eL${YAGIx(b4VN*~nKn6E~SIQUuRH zQ+5zP6jfnP$S0iJ@~t!Ai3o`X7biohli;E zT#yXyl{bojG@-TGZzpdVDXhbmF%F9+-^YSIv|MT1l3j zrxOFq>gd2%U}?6}8mIj?M zc077Zc9fq(-)4+gXv?Az26IO6eV`RAJz8e3)SC7~>%rlzDwySVx*q$ygTR5kW2ds- z!HBgcq0KON9*8Ff$X0wOq$`T7ml(@TF)VeoF}x1OttjuVHn3~sHrMB++}f7f9H%@f z=|kP_?#+fve@{0MlbkC9tyvQ_R?lRdRJ@$qcB(8*jyMyeME5ns6ypVI1Xm*Zr{DuS zZ!1)rQfa89c~;l~VkCiHI|PCBd`S*2RLNQM8!g9L6?n`^evQNEwfO@&JJRme+uopQX0%Jo zgd5G&#&{nX{o?TQwQvF1<^Cg3?2co;_06=~Hcb6~4XWpNFL!WU{+CK;>gH%|BLOh7@!hsa(>pNDAmpcuVO-?;Bic17R}^|6@8DahH)G z!EmhsfunLL|3b=M0MeK2vqZ|OqUqS8npxwge$w-4pFVXFq$_EKrZY?BuP@Az@(k`L z`ViQBSk`y+YwRT;&W| z2e3UfkCo^uTA4}Qmmtqs+nk#gNr2W4 zTH%hhErhB)pkXR{B!q5P3-OM+M;qu~f>}IjtF%>w{~K-0*jPVLl?Chz&zIdxp}bjx zStp&Iufr58FTQ36AHU)0+CmvaOpKF;W@sMTFpJ`j;3d)J_$tNQI^c<^1o<49Z(~K> z;EZTBaVT%14(bFw2ob@?JLQ2@(1pCdg3S%E4*dJ}dA*v}_a4_P(a`cHnBFJxNobAv zf&Zl-Yt*lhn-wjZsq<9v-IsXxAxMZ58C@e0!rzhJ+D@9^3~?~yllY^s$?&oNwyH!#~6x4gUrfxplCvK#!f z$viuszW>MFEcFL?>ux*((!L$;R?xc*myjRIjgnQX79@UPD$6Dz0jutM@7h_pq z0Zr)#O<^y_K6jfY^X%A-ip>P%3saX{!v;fxT-*0C_j4=UMH+Xth(XVkVGiiKE#f)q z%Jp=JT)uy{&}Iq2E*xr4YsJ5>w^=#-mRZ4vPXpI6q~1aFwi+lQcimO45V-JXP;>(Q zo={U`{=_JF`EQj87Wf}{Qy35s8r1*9Mxg({CvOt}?Vh9d&(}iI-quvs-rm~P;eRA@ zG5?1HO}puruc@S{YNAF3vmUc2B4!k*yi))<5BQmvd3tr}cIs#9)*AX>t`=~{f#Uz0 z0&Nk!7sSZwJe}=)-R^$0{yeS!V`Dh7w{w5rZ9ir!Z7Cd7dwZcK;BT#V0bzTt>;@Cl z#|#A!-IL6CZ@eHH!CG>OO8!%G8&8t4)Ro@}USB*k>oEUo0LsljsJ-%5Mo^MJF2I8- z#v7a5VdJ-Cd%(a+y6QwTmi+?f8Nxtm{g-+WGL>t;s#epv7ug>inqimZCVm!uT5Pf6 ziEgQt7^%xJf#!aPWbuC_3Nxfb&CFbQy!(8ANpkWLI4oSnH?Q3f?0k1t$3d+lkQs{~(>06l&v|MpcFsyAv zin6N!-;pggosR*vV=DO(#+}4ps|5$`udE%Kdmp?G7B#y%H`R|i8skKOd9Xzx8xgR$>Zo2R2Ytktq^w#ul4uicxW#{ zFjG_RNlBroV_n;a7U(KIpcp*{M~e~@>Q#Av90Jc5v%0c>egEdY4v3%|K1XvB{O_8G zkTWLC>OZKf;XguMH2-Pw{BKbFzaY;4v2seZV0>^7Q~d4O=AwaPhP3h|!hw5aqOtT@ z!SNz}$of**Bl3TK209@F=Tn1+mgZa8yh(Png%Zd6Mt}^NSjy)etQrF zme*llAW=N_8R*O~d2!apJnF%(JcN??=`$qs3Y+~xs>L9x`0^NIn!8mMRFA_tg`etw z3k{9JAjnl@ygIiJcNHTy02GMAvBVqEss&t2<2mnw!; zU`J)0>lWiqVqo|ex7!+@0i>B~BSU1A_0w#Ee+2pJx0BFiZ7RDHEvE*ptc9md(B{&+ zKE>TM)+Pd>HEmdJao7U@S>nL(qq*A)#eLOuIfAS@j`_sK0UEY6OAJJ-kOrHG zjHx`g!9j*_jRcJ%>CE9K2MVf?BUZKFHY?EpV6ai7sET-tqk=nDFh-(65rhjtlKEY% z@G&cQ<5BKatfdA1FKuB=i>CCC5(|9TMW%K~GbA4}80I5%B}(gck#Wlq@$nO3%@QP_ z8nvPkJFa|znk>V92cA!K1rKtr)skHEJD;k8P|R8RkCq1Rh^&}Evwa4BUJz2f!2=MH zo4j8Y$YL2313}H~F7@J7mh>u%556Hw0VUOz-Un@ZASCL)y8}4XXS`t1AC*^>PLwIc zUQok5PFS=*#)Z!3JZN&eZ6ZDP^-c@StY*t20JhCnbMxXf=LK#;`4KHEqMZ-Ly9KsS zI2VUJGY&PmdbM+iT)zek)#Qc#_i4uH43 z@T5SZBrhNCiK~~esjsO9!qBpaWK<`>!-`b71Y5ReXQ4AJU~T2Njri1CEp5oKw;Lnm)-Y@Z3sEY}XIgSy%xo=uek(kAAH5MsV$V3uTUsoTzxp_rF=tx zV07vlJNKtJhCu`b}*#m&5LV4TAE&%KtHViDAdv#c^x`J7bg z&N;#I2GkF@SIGht6p-V}`!F_~lCXjl1BdTLIjD2hH$J^YFN`7f{Q?OHPFEM$65^!u zNwkelo*5+$ZT|oQ%o%;rBX$+?xhvjb)SHgNHE_yP%wYkkvXHS{Bf$OiKJ5d1gI0j< zF6N}Aq=(WDo(J{e-uOecxPD>XZ@|u-tgTR<972`q8;&ZD!cep^@B5CaqFz|oU!iFj zU0;6fQX&~15E53EW&w1s9gQQ~Zk16X%6 zjG`j0yq}4deX2?Tr(03kg>C(!7a|b9qFI?jcE^Y>-VhudI@&LI6Qa}WQ>4H_!UVyF z((cm&!3gmq@;BD#5P~0;_2qgZhtJS|>WdtjY=q zLnHH~Fm!cxw|Z?Vw8*~?I$g#9j&uvgm7vPr#&iZgPP~v~BI4jOv;*OQ?jYJtzO<^y z7-#C={r7CO810!^s(MT!@@Vz_SVU)7VBi(e1%1rvS!?PTa}Uv`J!EP3s6Y!xUgM^8 z4f!fq<3Wer_#;u!5ECZ|^c1{|q_lh3m^9|nsMR1#Qm|?4Yp5~|er2?W^7~cl;_r4WSme_o68J9p03~Hc%X#VcX!xAu%1`R!dfGJCp zV*&m47>s^%Ib0~-2f$6oSgn3jg8m%UA;ArcdcRyM5;}|r;)?a^D*lel5C`V5G=c~k zy*w_&BfySOxE!(~PI$*dwG><+-%KT5p?whOUMA*k<9*gi#T{h3DAxzAPxN&Xws8o9Cp*`PA5>d9*Z-ynV# z9yY*1WR^D8|C%I@vo+d8r^pjJ$>eo|j>XiLWvTWLl(^;JHCsoPgem6PvegHb-OTf| zvTgsHSa;BkbG=(NgPO|CZu9gUCGr$8*EoH2_Z#^BnxF0yM~t`|9ws_xZ8X8iZYqh! zAh;HXJ)3P&)Q0(&F>!LN0g#bdbis-cQxyGn9Qgh`q+~49Fqd2epikEUw9caM%V6WgP)532RMRW}8gNS%V%Hx7apSz}tn@bQy!<=lbhmAH=FsMD?leawbnP5BWM0 z5{)@EEIYMu5;u)!+HQWhQ;D3_Cm_NADNeb-f56}<{41aYq8p4=93d=-=q0Yx#knGYfXVt z+kMxlus}t2T5FEyCN~!}90O_X@@PQpuy;kuGz@bWft%diBTx?d)_xWd_-(!LmVrh**oKg!1CNF&LX4{*j|) zIvjCR0I2UUuuEXh<9}oT_zT#jOrJAHNLFT~Ilh9hGJPI1<5`C-WA{tUYlyMeoy!+U zhA#=p!u1R7DNg9u4|QfED-2TuKI}>p#2P9--z;Bbf4Op*;Q9LCbO&aL2i<0O$ByoI z!9;Ght733FC>Pz>$_mw(F`zU?`m@>gE`9_p*=7o=7av`-&ifU(^)UU`Kg3Kw`h9-1 z6`e6+im=|m2v`pN(2dE%%n8YyQz;#3Q-|x`91z?gj68cMrHl}C25|6(_dIGk*8cA3 zRHB|Nwv{@sP4W+YZM)VKI>RlB`n=Oj~Rzx~M+Khz$N$45rLn6k1nvvD^&HtsMA4`s=MmuOJID@$s8Ph4E zAmSV^+s-z8cfv~Yd(40Sh4JG#F~aB>WFoX7ykaOr3JaJ&Lb49=B8Vk-SQT9%7TYhv z?-Pprt{|=Y5ZQ1?od|A<_IJU93|l4oAfBm?3-wk{O<8ea+`}u%(kub(LFo2zFtd?4 zwpN|2mBNywv+d^y_8#<$r>*5+$wRTCygFLcrwT(qc^n&@9r+}Kd_u@Ithz(6Qb4}A zWo_HdBj#V$VE#l6pD0a=NfB0l^6W^g`vm^sta>Tly?$E&{F?TTX~DsKF~poFfmN%2 z4x`Dc{u{Lkqz&y!33;X}weD}&;7p>xiI&ZUb1H9iD25a(gI|`|;G^NwJPv=1S5e)j z;U;`?n}jnY6rA{V^ zxTd{bK)Gi^odL3l989DQlN+Zs39Xe&otGeY(b5>rlIqfc7Ap4}EC?j<{M=hlH{1+d zw|c}}yx88_xQr`{98Z!d^FNH77=u(p-L{W6RvIn40f-BldeF-YD>p6#)(Qzf)lfZj z?3wAMtPPp>vMehkT`3gToPd%|D8~4`5WK{`#+}{L{jRUMt zrFz+O$C7y8$M&E4@+p+oV5c%uYzbqd2Y%SSgYy#xh4G3hQv>V*BnuKQhBa#=oZB~w{azUB+q%bRe_R^ z>fHBilnRTUfaJ201czL8^~Ix#+qOHSO)A|xWLqOxB$dT2W~)e-r9;bm=;p;RjYahB z*1hegN(VKK+ztr~h1}YP@6cfj{e#|sS`;3tJhIJK=tVJ-*h-5y9n*&cYCSdg#EHE# zSIx=r#qOaLJoVVf6v;(okg6?*L_55atl^W(gm^yjR?$GplNP>BZsBYEf_>wM0Lc;T zhf&gpzOWNxS>m+mN92N0{;4uw`P+9^*|-1~$uXpggj4- z^SFc4`uzj2OwdEVT@}Q`(^EcQ_5(ZtXTql*yGzdS&vrS_w>~~ra|Nb5abwf}Y!uq6R5f&6g2ge~2p(%c< z@O)cz%%rr4*cRJ5f`n@lvHNk@lE1a*96Kw6lJ~B-XfJW%?&-y?;E&?1AacU@`N`!O z6}V>8^%RZ7SQnZ-z$(jsX`amu*5Fj8g!3RTRwK^`2_QHe;_2y_n|6gSaGyPmI#kA0sYV<_qOZc#-2BO%hX)f$s-Z3xlI!ub z^;3ru11DA`4heAu%}HIXo&ctujzE2!6DIGE{?Zs>2}J+p&C$rc7gJC35gxhflorvsb%sGOxpuWhF)dL_&7&Z99=5M0b~Qa;Mo!j&Ti_kXW!86N%n= zSC@6Lw>UQ__F&+&Rzv?gscwAz8IP!n63>SP)^62(HK98nGjLY2*e^OwOq`3O|C92? z;TVhZ2SK%9AGW4ZavTB9?)mUbOoF`V7S=XM;#3EUpR+^oHtdV!GK^nXzCu>tpR|89 zdD{fnvCaN^^LL%amZ^}-E+214g&^56rpdc@yv0b<3}Ys?)f|fXN4oHf$six)-@<;W&&_kj z-B}M5U*1sb4)77aR=@%I?|Wkn-QJVuA96an25;~!gq(g1@O-5VGo7y&E_srxL6ZfS z*R%$gR}dyONgju*D&?geiSj7SZ@ftyA|}(*Y4KbvU!YLsi1EDQQCnb+-cM=K1io78o!v*);o<XwjaQH%)uIP&Zm?)Nfbfn;jIr z)d#!$gOe3QHp}2NBak@yYv3m(CPKkwI|{;d=gi552u?xj9ObCU^DJFQp4t4e1tPzM zvsRIGZ6VF+{6PvqsplMZWhz10YwS={?`~O0Ec$`-!klNUYtzWA^f9m7tkEzCy<_nS z=&<(awFeZvt51>@o_~>PLs05CY)$;}Oo$VDO)?l-{CS1Co=nxjqben*O1BR>#9`0^ zkwk^k-wcLCLGh|XLjdWv0_Hg54B&OzCE^3NCP}~OajK-LuRW53CkV~Su0U>zN%yQP zH8UH#W5P3-!ToO-2k&)}nFe`t+mdqCxxAHgcifup^gKpMObbox9LFK;LP3}0dP-UW z?Zo*^nrQ6*$FtZ(>kLCc2LY*|{!dUn$^RW~m9leoF|@Jy|M5p-G~j%+P0_#orRKf8 zvuu5<*XO!B?1E}-*SY~MOa$6c%2cM+xa8}_8x*aVn~57v&W(0mqN1W`5a7*VN{SUH zXz98DDyCnX2EPl-`Lesf`=AQT%YSDb`$%;(jUTrNen$NPJrlpPDP}prI>Ml!r6bCT;mjsg@X^#&<}CGf0JtR{Ecwd&)2zuhr#nqdgHj+g2n}GK9CHuwO zk>oZxy{vcOL)$8-}L^iVfJHAGfwN$prHjYV0ju}8%jWquw>}_W6j~m<}Jf!G?~r5&Rx)!9JNX!ts#SGe2HzobV5); zpj@&`cNcO&q+%*<%D7za|?m5qlmFK$=MJ_iv{aRs+BGVrs)98BlN^nMr{V_fcl_;jkzRju+c-y?gqBC_@J0dFLq-D9@VN&-`R9U;nv$Hg?>$oe4N&Ht$V_(JR3TG^! zzJsbQbi zFE6-{#9{G{+Z}ww!ycl*7rRdmU#_&|DqPfX3CR1I{Kk;bHwF6jh0opI`UV2W{*|nn zf_Y@%wW6APb&9RrbEN=PQRBEpM(N1w`81s=(xQj6 z-eO0k9=Al|>Ej|Mw&G`%q8e$2xVz1v4DXAi8G};R$y)ww638Y=9y$ZYFDM$}vzusg zUf+~BPX>(SjA|tgaFZr_e0{)+z9i6G#lgt=F_n$d=beAt0Sa0a7>z-?vcjl3e+W}+ z1&9=|vC=$co}-Zh*%3588G?v&U7%N1Qf-wNWJ)(v`iO5KHSkC5&g7CrKu8V}uQGcfcz zmBz#Lbqwqy#Z~UzHgOQ;Q-rPxrRNvl(&u6ts4~0=KkeS;zqURz%!-ERppmd%0v>iRlEf+H$yl{_8TMJzo0 z>n)`On|7=WQdsqhXI?#V{>+~}qt-cQbokEbgwV3QvSP7&hK4R{Z{aGHVS3;+h{|Hz z6$Js}_AJr383c_+6sNR|$qu6dqHXQTc6?(XWPCVZv=)D#6_;D_8P-=zOGEN5&?~8S zl5jQ?NL$c%O)*bOohdNwGIKM#jSAC?BVY={@A#c9GmX0=T(0G}xs`-%f3r=m6-cpK z!%waekyAvm9C3%>sixdZj+I(wQlbB4wv9xKI*T13DYG^T%}zZYJ|0$Oj^YtY+d$V$ zAVudSc-)FMl|54n=N{BnZTM|!>=bhaja?o7s+v1*U$!v!qQ%`T-6fBvmdPbVmro&d zk07TOp*KuxRUSTLRrBj{mjsnF8`d}rMViY8j`jo~Hp$fkv9F_g(jUo#Arp;Xw0M$~ zRIN!B22~$kx;QYmOkos@%|5k)!QypDMVe}1M9tZfkpXKGOxvKXB!=lo`p?|R1l=tA zp(1}c6T3Fwj_CPJwVsYtgeRKg?9?}%oRq0F+r+kdB=bFUdVDRPa;E~~>2$w}>O>v=?|e>#(-Lyx?nbg=ckJ#5U6;RT zNvHhXk$P}m9wSvFyU3}=7!y?Y z=fg$PbV8d7g25&-jOcs{%}wTDKm>!Vk);&rr;O1nvO0VrU&Q?TtYVU=ir`te8SLlS zKSNmV=+vF|ATGg`4$N1uS|n??f}C_4Sz!f|4Ly8#yTW-FBfvS48Tef|-46C(wEO_%pPhUC5$-~Y?!0vFZ^Gu`x=m7X99_?C-`|h zfmMM&Y@zdfitA@KPw4Mc(YHcY1)3*1xvW9V-r4n-9ZuBpFcf{yz+SR{ zo$ZSU_|fgwF~aakGr(9Be`~A|3)B=9`$M-TWKipq-NqRDRQc}ABo*s_5kV%doIX7LRLRau_gd@Rd_aLFXGSU+U?uAqh z8qusWWcvgQ&wu{|sRXmv?sl=xc<$6AR$+cl& zFNh5q1~kffG{3lDUdvEZu5c(aAG~+64FxdlfwY^*;JSS|m~CJusvi-!$XR`6@XtY2 znDHSz7}_Bx7zGq-^5{stTRy|I@N=>*y$zz>m^}^{d&~h;0kYiq8<^Wq7Dz0w31ShO^~LUfW6rfitR0(=3;Uue`Y%y@ex#eKPOW zO~V?)M#AeHB2kovn1v=n^D?2{2jhIQd9t|_Q+c|ZFaWt+r&#yrOu-!4pXAJuxM+Cx z*H&>eZ0v8Y`t}8{TV6smOj=__gFC=eah)mZt9gwz>>W$!>b3O;Rm^Ig*POZP8Rl0f zT~o=Nu1J|lO>}xX&#P58%Yl z83`HRs5#32Qm9mdCrMlV|NKNC+Z~ z9OB8xk5HJ>gBLi+m@(pvpw)1(OaVJKs*$Ou#@Knd#bk+V@y;YXT?)4eP9E5{J%KGtYinNYJUH9PU3A}66c>Xn zZ{Bn0<;8$WCOAL$^NqTjwM?5d=RHgw3!72WRo0c;+houoUA@HWLZM;^U$&sycWrFd zE7ekt9;kb0`lps{>R(}YnXlyGY}5pPd9zBpgXeJTY_jwaJGSJQC#-KJqmh-;ad&F- z-Y)E>!&`Rz!HtCz>%yOJ|v(u7P*I$jqEY3}(Z-orn4 zlI?CYKNl`6I){#2P1h)y(6?i;^z`N3bxTV%wNvQW+eu|x=kbj~s8rhCR*0H=iGkSj zk23lr9kr|p7#qKL=UjgO`@UnvzU)`&fI>1Qs7ubq{@+lK{hH* zvl6eSb9%yngRn^T<;jG1SVa)eA>T^XX=yUS@NCKpk?ovCW1D@!=@kn;l_BrG;hOTC z6K&H{<8K#dI(A+zw-MWxS+~{g$tI7|SfP$EYKxA}LlVO^sT#Oby^grkdZ^^lA}uEF zBSj$weBJG{+Bh@Yffzsw=HyChS(dtLE3i*}Zj@~!_T-Ay7z=B)+*~3|?w`Zd)Co2t zC&4DyB!o&YgSw+fJn6`sn$e)29`kUwAc+1MND7YjV%lO;H2}fNy>hD#=gT ze+-aFNpyKIoXY~Vq-}OWPBe?Rfu^{ps8>Xy%42r@RV#*QV~P83jdlFNgkPN=T|Kt7 zV*M`Rh*30&AWlb$;ae130e@}Tqi3zx2^JQHpM>j$6x`#{mu%tZlwx9Gj@Hc92IuY* zarmT|*d0E~vt6<+r?W^UW0&#U&)8B6+1+;k^2|FWBRP9?C4Rk)HAh&=AS8FS|NQaZ z2j!iZ)nbEyg4ZTp-zHwVlfLC~tXIrv(xrP8PAtR{*c;T24ycA-;auWsya-!kF~CWZ zw_uZ|%urXgUbc@x=L=_g@QJ@m#5beS@6W195Hn7>_}z@Xt{DIEA`A&V82bc^#!q8$ zFh?z_Vn|ozJ;NPd^5uu(9tspo8t%&-U9Ckay-s@DnM*R5rtu|4)~e)`z0P-sy?)kc zs_k&J@0&0!q4~%cKL)2l;N*T&0;mqX5T{Qy60%JtKTQZ-xb%KOcgqwJmb%MOOKk7N zgq})R_6**{8A|6H?fO+2`#QU)p$Ei2&nbj6TpLSIT^D$|`TcSeh+)}VMb}LmvZ{O| ze*1IdCt3+yhdYVxcM)Q_V0bIXLgr6~%JS<<&dxIgfL=Vnx4YHuU@I34JXA|+$_S3~ zy~X#gO_X!cSs^XM{yzDGNM>?v(+sF#<0;AH^YrE8smx<36bUsHbN#y57K8WEu(`qHvQ6cAZPo=J5C(lSmUCZ57Rj6cx!e^rfaI5%w}unz}4 zoX=nt)FVNV%QDJH`o!u9olLD4O5fl)xp+#RloZlaA92o3x4->?rB4`gS$;WO{R;Z3>cG3IgFX2EA?PK^M}@%1%A;?f6}s&CV$cIyEr#q5;yHdNZ9h{| z-=dX+a5elJoDo?Eq&Og!nN6A)5yYpnGEp}?=!C-V)(*~z-+?kY1Q7qs#Rsy%hu_60rdbB+QQNr?S1 z?;xtjUv|*E3}HmuNyB9aFL5H~3Ho0UsmuMZELp1a#CA1g`P{-mT?BchuLEtK}!QZ=3AWakRu~?f9V~3F;TV`5%9Pcs_$gq&CcU}r8gOO zC2&SWPsSG{&o-LIGTBqp6SLQZPvYKp$$7L4WRRZ0BR$Kf0I0SCFkqveCp@f)o8W)! z$%7D1R`&j7W9Q9CGus_)b%+B#J2G;l*FLz#s$hw{BHS~WNLODV#(!u_2Pe&tMsq={ zdm7>_WecWF#D=?eMjLj=-_z`aHMZ=3_-&E8;ibPmM}61i6J3is*=dKf%HC>=xbj4$ zS|Q-hWQ8T5mWde6h@;mS+?k=89?1FU<%qH9B(l&O>k|u_aD|DY*@~(`_pb|B#rJ&g zR0(~(68fpUPz6TdS@4JT5MOPrqDh5_H(eX1$P2SQrkvN8sTxwV>l0)Qq z0pzTuvtEAKRDkKGhhv^jk%|HQ1DdF%5oKq5BS>szk-CIke{%js?~%@$uaN3^Uz6Wf z_iyx{bZ(;9y4X&>LPV=L=d+A}7I4GkK0c1Xts{rrW1Q7apHf-))`BgC^0^F(>At1* za@e7{lq%yAkn*NH8Q1{@{lKhRg*^TfGvv!Sn*ed*x@6>M%aaqySxR|oNadYt1mpUZ z6H(rupHYf&Z z29$5g#|0MX#aR6TZ$@eGxxABRKakDYtD%5BmKp;HbG_ZbT+=81E&=XRk6m_3t9PvD zr5Cqy(v?gHcYvYvXkNH@S#Po~q(_7MOuCAB8G$a9BC##gw^5mW16cML=T=ERL7wsk zzNEayTG?mtB=x*wc@ifBCJ|irFVMOvH)AFRW8WE~U()QT=HBCe@s$dA9O!@`zAAT) zaOZ7l6vyR+Nk_OOF!ZlZmjoImKh)dxFbbR~z(cMhfeX1l7S_`;h|v3gI}n9$sSQ>+3@AFAy9=B_y$)q;Wdl|C-X|VV3w8 z2S#>|5dGA8^9%Bu&fhmVRrTX>Z7{~3V&0UpJNEl0=N32euvDGCJ>#6dUSi&PxFW*s zS`}TB>?}H(T2lxBJ!V#2taV;q%zd6fOr=SGHpoSG*4PDaiG0pdb5`jelVipkEk%FV zThLc@Hc_AL1#D&T4D=w@UezYNJ%0=f3iVRuVL5H?eeZM}4W*bomebEU@e2d`M<~uW zf#Bugwf`VezG|^Qbt6R_=U0}|=k;mIIakz99*>FrsQR{0aQRP6ko?5<7bkDN8evZ& zB@_KqQG?ErKL=1*ZM9_5?Pq%lcS4uLSzN(Mr5=t6xHLS~Ym`UgM@D&VNu8e?_=nSFtF$u@hpPSmI4Vo_t&v?>$~K4y(O~Rb*(MFy_igM7 z*~yYUyR6yQgzWnWMUgDov!!g=lInM+=lOmOk4L`O?{i&qxy&D*_qorRbDwj6?)!ef z#JLd7F6Z2I$S0iYI={rZNk*<{HtIl^mx=h>Cim*04K4+Z4IJtd*-)%6XV2(MCscPiw_a+y*?BKbTS@BZ3AUao^%Zi#PhoY9Vib4N>SE%4>=Jco0v zH_Miey{E;FkdlZSq)e<{`+S3W=*ttvD#hB8w=|2aV*D=yOV}(&p%0LbEWH$&@$X3x~CiF-?ejQ*N+-M zc8zT@3iwkdRT2t(XS`d7`tJQAjRmKAhiw{WOqpuvFp`i@Q@!KMhwKgsA}%@sw8Xo5Y=F zhRJZg)O4uqNWj?V&&vth*H#je6T}}p_<>!Dr#89q@uSjWv~JuW(>FqoJ5^ho0%K?E z9?x_Q;kmcsQ@5=}z@tdljMSt9-Z3xn$k)kEjK|qXS>EfuDmu(Z8|(W?gY6-l z@R_#M8=vxKMAoi&PwnaIYw2COJM@atcgfr=zK1bvjW?9B`-+Voe$Q+H$j!1$Tjn+* z&LY<%)L@;zhnJlB^Og6I&BOR-m?{IW;tyYC%FZ!&Z>kGjHJ6cqM-F z&19n+e1=9AH1VrVeHrIzqlC`w9=*zfmrerF?JMzO&|Mmv;!4DKc(sp+jy^Dx?(8>1 zH&yS_4yL7m&GWX~mdfgH*AB4{CKo;+egw=PrvkTaoBU+P-4u?E|&!c z)DKc;>$$B6u*Zr1SjUh2)FeuWLWHl5TH(UHWkf zLs>7px!c5n;rbe^lO@qlYLzlDVp(z?6rPZel=YB)Uv&n!2{+Mb$-vQl=xKw( zve&>xYx+jW_NJh!FV||r?;hdP*jOXYcLCp>DOtJ?2S^)DkM{{Eb zS$!L$e_o0(^}n3tA1R3-$SNvgBq;DOEo}fNc|tB%%#g4RA3{|euq)p+xd3I8^4E&m zFrD%}nvG^HUAIKe9_{tXB;tl|G<%>yk6R;8L2)KUJw4yHJXUOPM>(-+jxq4R;z8H#>rnJy*)8N+$wA$^F zN+H*3t)eFEgxLw+Nw3};4WV$qj&_D`%ADV2%r zJCPCo%{=z7;`F98(us5JnT(G@sKTZ^;2FVitXyLe-S5(hV&Ium+1pIUB(CZ#h|g)u zSLJJ<@HgrDiA-}V_6B^x1>c9B6%~847JkQ!^KLZ2skm;q*edo;UA)~?SghG8;QbHh z_6M;ouo_1rq9=x$<`Y@EA{C%6-pEV}B(1#sDoe_e1s3^Y>n#1Sw;N|}8D|s|VPd+g z-_$QhCz`vLxxrVMx3ape1xu3*wjx=yKSlM~nFgkNWb4?DDr*!?U)L_VeffF<+!j|b zZ$Wn2$TDv3C3V@BHpSgv3JUif8%hk%OsGZ=OxH@8&4`bbf$`aAMchl^qN>Eyu3JH} z9-S!x8-s4fE=lad%Pkp8hAs~u?|uRnL48O|;*DEU! zuS0{cpk%1E0nc__2%;apFsTm0bKtd&A0~S3Cj^?72-*Owk3V!ZG*PswDfS~}2<8le z5+W^`Y(&R)yVF*tU_s!XMcJS`;(Tr`J0%>p=Z&InR%D3@KEzzI+-2)HK zuoNZ&o=wUC&+*?ofPb0a(E6(<2Amd6%uSu_^-<1?hsxs~0K5^f(LsGqgEF^+0_H=uNk9S0bb!|O8d?m5gQjUKevPaO+*VfSn^2892K~%crWM8+6 z25@V?Y@J<9w%@NXh-2!}SK_(X)O4AM1-WTg>sj1{lj5@=q&dxE^9xng1_z9w9DK>| z6Iybcd0e zyi;Ew!KBRIfGPGytQ6}z}MeXCfLY0?9%RiyagSp_D1?N&c{ zyo>VbJ4Gy`@Fv+5cKgUgs~na$>BV{*em7PU3%lloy_aEovR+J7TfQKh8BJXyL6|P8un-Jnq(ghd!_HEOh$zlv2$~y3krgeH;9zC}V3f`uDtW(%mT#944DQa~^8ZI+zAUu4U(j0YcDfKR$bK#gvn_{JZ>|gZ5+)u?T$w7Q%F^;!Wk?G z(le7r!ufT*cxS}PR6hIVtXa)i`d$-_1KkyBU>qmgz-=T};uxx&sKgv48akIWQ89F{ z0XiY?WM^~;|T8zBOr zs#zuOONzH?svv*jokd5SK8wG>+yMC)LYL|vLqm^PMHcT=`}V$=nIRHe2?h)8WQa6O zPAU}d`1y(>kZiP~Gr=mtJLMu`i<2CspL|q2DqAgAD^7*$xzM`PU4^ga`ilE134XBQ z99P(LhHU@7qvl9Yzg$M`+dlS=x^(m-_3t|h>S}E0bcFMn=C|KamQ)=w2^e)35p`zY zRV8X?d;s^>Cof2SPR&nP3E+-LCkS0J$H!eh8~k0qo$}00b=7!H_I2O+Ro@3O$nPdm ztmbOO^B+IHzQ5w>@@@J4cKw5&^_w6s!s=H%&byAbUtczPQ7}wfTqxxtQNfn*u73Qw zGuWsrky_ajPx-5`R<)6xHf>C(oqGf_Fw|-U*GfS?xLML$kv;h_pZ@Kk$y0X(S+K80 z6^|z)*`5VUkawg}=z`S;VhZhxyDfrE0$(PMurAxl~<>lfZa>JZ288ULK7D` zl9|#L^JL}Y$j*j`0-K6kH#?bRmg#5L3iB4Z)%iF@SqT+Lp|{i`m%R-|ZE94Np7Pa5 zCqC^V3}B(FR340pmF*qaa}M}+h6}mqE~7Sh!9bDv9YRT|>vBNAqv09zXHMlcuhKD| zcjjA(b*XCIwJ33?CB!+;{)vX@9xns_b-VO{i0y?}{!sdXj1GM8+$#v>W7nw;+O_9B z_{4L;C6ol?(?W0<6taGEn1^uG=?Q3i29sE`RfYCaV$3DKc_;?HsL?D_fSYg}SuO5U zOB_f4^vZ_x%o`5|C@9C5+o=mFy@au{s)sKw!UgC&L35aH(sgDxRE2De%(%OT=VUdN ziVLEmdOvJ&5*tCMKRyXctCwQu_RH%;m*$YK&m;jtbdH#Ak~13T1^f89tn`A%QEHWs~jnY~E}p_Z$XC z=?YXLCkzVSK+Id`xZYTegb@W8_baLt-Fq`Tv|=)JPbFsKRm)4UW;yT+J`<)%#ue9DPOkje)YF2fsCilK9MIIK>p*`fkoD5nGfmLwt)!KOT+> zOFq*VZktDDyM3P5UOg`~XL#cbzC}eL%qMB=Q5$d89MKuN#$6|4gx_Jt0Gfn8w&q}%lq4QU%6#jT*MRT% zrLz~C8FYKHawn-EQWN1B75O&quS+Z81(zN)G>~vN8VwC+e+y(`>HcxC{MrJ;H1Z4k zZWuv$w_F0-Ub%MVcpIc){4PGL^I7M{>;hS?;eH!;gmcOE66z3;Z1Phqo(t zVP(Hg6q#0gIKgsg7L7WE!{Y#1nI(45tx2{$34dDd#!Z0NIyrm)HOn5W#7;f4pQci# zDW!FI(g4e668kI9{2+mLwB+=#9bfqgX%!B34V-$wwSN(_cm*^{y0jQtv*4}eO^sOV z*9xoNvX)c9isB}Tgx&ZRjp3kwhTVK?r9;n!x>^XYT z@Q^7zp{rkIs{2mUSE^2!Gf6$6;j~&4=-0cSJJDizZp6LTe8b45;{AKM%v99}{{FfC zz709%u0mC=1KXTo(=TqmZQ;c?$M3z(!xah>aywrj40sc2y3rKFw4jCq+Y+u=CH@_V zxz|qeTwa>+<|H%8Dz5u>ZI5MmjTFwXS-Fv!TDd*`>3{krWoNVx$<133`(ftS?ZPyY z&4@ah^3^i`vL$BZa>O|Nt?ucewzsF)0zX3qmM^|waXr=T0pfIb0*$AwU=?Ipl|1Y; z*Pk6{C-p4MY;j@IJ|DW>QHZQJcp;Z~?8(Q+Kk3^0qJ}SCk^*n4W zu9ZFwLHUx-$6xvaQ)SUQcYd6fF8&x)V`1bIuX@>{mE$b|Yd(qomn3;bPwnDUc0F=; zh*6_((%bqAYQWQ~odER?h>1mkL4kpb3s7`0m@rDKGU*oyF)$j~Ffd4fXV$?`f~rHf zB%Y)@5SXZvfwm10RY5X?TEo)PK_`L6qgBp=#>fO49$D zDq8Ozj0q6213tV5Qq=;fZ0$|KroY{Dz=l@lU^J)?Ko@ti20TRplXzphBi>XGx4bou zEWrkNjz0t5j!_ke{g5I#PUlEU$Km8g8TE|XK=MkU@PT4T><2OVamoK;wJ}3X0L$vX zgd7gNa359*nc)R-0!`2X@FOTB`+oETOPc=ubp5R)VQgY+5BTZZJ2?9QwnO=dnulIUF3gFn;BODC2)65)HeVd%t86sL7Rv^Y+nbn+&l z6BAJY(ETvwI)Ts$aiE8rht4KD*qNyE{8{x6R|%akbTBzw;2+6Echkt+W+`u^XX z_z&x%n Date: Fri, 15 Aug 2025 17:21:32 -0700 Subject: [PATCH 43/81] java slowe --- test-server/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-server/Makefile b/test-server/Makefile index 4941f3d1..52deb115 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -30,7 +30,7 @@ start-servers: AWS_REGION="us-west-2" \ ./gradlew run & echo $$! > ../java-server.pid @echo "Waiting for servers to be ready..." - @for i in $$(seq 1 60); do \ + @for i in $$(seq 1 360); do \ if nc -z localhost 8080 && nc -z localhost 8081; then \ echo "Ports are open, waiting for servers to initialize..."; \ sleep 5; \ From b3fa01eac1146ead889442539f07a9b8133eb360 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 15 Aug 2025 17:23:36 -0700 Subject: [PATCH 44/81] im slowe --- test-server/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-server/Makefile b/test-server/Makefile index 52deb115..146a53d8 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -37,11 +37,11 @@ start-servers: echo "Both servers are ready!"; \ break; \ fi; \ - if [ $$i -eq 60 ]; then \ + if [ $$i -eq 360 ]; then \ echo "Timeout waiting for servers to start"; \ exit 1; \ fi; \ - echo "Waiting for servers to start ($$i/60)..."; \ + echo "Waiting for servers to start ($$i/360)..."; \ sleep 1; \ done From c03a7758394b4cda0cc68c7cd6618f53774eb2e5 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 18 Aug 2025 11:04:18 -0700 Subject: [PATCH 45/81] black --- test-server/python-server/src/main.py | 141 ++++++++++++++------------ 1 file changed, 75 insertions(+), 66 deletions(-) diff --git a/test-server/python-server/src/main.py b/test-server/python-server/src/main.py index a7ccbdea..05405d69 100755 --- a/test-server/python-server/src/main.py +++ b/test-server/python-server/src/main.py @@ -1,6 +1,7 @@ """ Main entry point for the Python server. """ + from fastapi import FastAPI, Request, HTTPException, Response, status from fastapi.responses import JSONResponse from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig @@ -16,11 +17,12 @@ # Dictionary to store clients with their UUIDs as keys client_cache = {} -# Java gets a list, but since there's no Smithy Python Server, + +# Java gets a list, but since there's no Smithy Python Server, # this is just a string. def metadata_string_to_map(md_string): md = {} - if md_string == '': + if md_string == "": return md md_list = md_string.split(",") for entry in md_list: @@ -36,20 +38,22 @@ def metadata_string_to_map(md_string): return md -def create_generic_server_error(message: str, status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR): +def create_generic_server_error( + message: str, status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR +): """ Create a response that matches the GenericServerError type from the Smithy model. Used for internal server errors. """ return JSONResponse( status_code=status_code, - content={ - "__type": "software.amazon.encryption.s3#GenericServerError", - "message": message - } + content={"__type": "software.amazon.encryption.s3#GenericServerError", "message": message}, ) -def create_s3_encryption_client_error(message: str, status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR): + +def create_s3_encryption_client_error( + message: str, status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR +): """ Create a response that matches the S3EncryptionClientError type from the Smithy model. Used for errors thrown by the S3 Encryption Client. @@ -58,10 +62,11 @@ def create_s3_encryption_client_error(message: str, status_code: int = status.HT status_code=status_code, content={ "__type": "software.amazon.encryption.s3#S3EncryptionClientError", - "message": message - } + "message": message, + }, ) + @app.put("/object/{bucket}/{key}") async def put_object(bucket: str, key: str, request: Request): """ @@ -72,41 +77,41 @@ async def put_object(bucket: str, key: str, request: Request): body = await request.body() print(f"PUT object request - Bucket: {bucket}, Key: {key}") print(f"ClientID from header: {client_id}") - + if not client_id: - return create_generic_server_error("ClientID header is required", status.HTTP_400_BAD_REQUEST) - + return create_generic_server_error( + "ClientID header is required", status.HTTP_400_BAD_REQUEST + ) + # Get the S3EncryptionClient from the client_cache client = client_cache.get(client_id) if not client: - return create_generic_server_error(f"No client found for ClientID: {client_id}", status.HTTP_404_NOT_FOUND) - + return create_generic_server_error( + f"No client found for ClientID: {client_id}", status.HTTP_404_NOT_FOUND + ) + try: - metadata = request.headers.get("Content-Metadata", '') + metadata = request.headers.get("Content-Metadata", "") enc_ctx = metadata_string_to_map(metadata) - + # Make the PutObject request response = client.put_object( - **{ - "Bucket": bucket, - "Key": key, - "Body": body, - "EncryptionContext": enc_ctx - } + **{"Bucket": bucket, "Key": key, "Body": body, "EncryptionContext": enc_ctx} ) - + print(f"PutObject response: {response}") - + # Return the appropriate response return { "bucket": bucket, "key": key, - "metadata": metadata if isinstance(metadata, list) else [] + "metadata": metadata if isinstance(metadata, list) else [], } except Exception as e: print(f"Error making PutObject request: {e}") return create_s3_encryption_client_error(f"Failed to put object: {str(e)}") + @app.get("/object/{bucket}/{key}") async def get_object(bucket: str, key: str, request: Request): """ @@ -116,54 +121,49 @@ async def get_object(bucket: str, key: str, request: Request): client_id = request.headers.get("ClientID") print(f"GET object request - Bucket: {bucket}, Key: {key}") print(f"ClientID from header: {client_id}") - + if not client_id: - return create_generic_server_error("ClientID header is required", status.HTTP_400_BAD_REQUEST) - + return create_generic_server_error( + "ClientID header is required", status.HTTP_400_BAD_REQUEST + ) + # Get the S3EncryptionClient from the client_cache client = client_cache.get(client_id) if not client: - return create_generic_server_error(f"No client found for ClientID: {client_id}", status.HTTP_404_NOT_FOUND) + return create_generic_server_error( + f"No client found for ClientID: {client_id}", status.HTTP_404_NOT_FOUND + ) - metadata = request.headers.get("Content-Metadata", '') + metadata = request.headers.get("Content-Metadata", "") enc_ctx = metadata_string_to_map(metadata) - + try: # Use the client to make a GetObject request to S3 print("making Get for " + key) - response = client.get_object( - **{ - "Bucket": bucket, - "Key": key, - "EncryptionContext": enc_ctx - } - ) - + response = client.get_object(**{"Bucket": bucket, "Key": key, "EncryptionContext": enc_ctx}) + print(f"GetObject response: {response}") - + # Extract the body and metadata from the response - body = response.get('Body').read() if response.get('Body') else b'' + body = response.get("Body").read() if response.get("Body") else b"" # print(f"body:" + body) - metadata = response.get('Metadata', []) + metadata = response.get("Metadata", []) print(f"md: {metadata}") - + # Convert metadata dictionary to a list of key-value pairs if it's a dict if isinstance(metadata, dict): metadata_list = [f"{key}={value}" for key, value in metadata.items()] else: metadata_list = metadata if isinstance(metadata, list) else [] - + # Set the Content-Metadata header in the response # Convert metadata_list to a comma-separated string metadata_str = ",".join(metadata_list) if metadata_list else "" headers = {"Content-Metadata": metadata_str} print(f"headers: {headers}") - + # Return the body as the response payload - return Response( - content=body, - headers=headers - ) + return Response(content=body, headers=headers) except S3EncryptionClientError as ex: print(f"Modeled Error making GetObject request: {ex}") return create_s3_encryption_client_error(str(ex)) @@ -171,6 +171,7 @@ async def get_object(bucket: str, key: str, request: Request): print(f"Generic Error making GetObject request: {e}") return create_generic_server_error(e) + @app.post("/client") async def client_endpoint(request: Request): """ @@ -178,13 +179,13 @@ async def client_endpoint(request: Request): """ body = await request.body() print(f"Received client request with body: {body}") - + # Parse the bytes object as JSON try: # Decode bytes to string and parse as JSON - parsed_data = json.loads(body.decode('utf-8')) + parsed_data = json.loads(body.decode("utf-8")) print(f"Parsed JSON data: {parsed_data}") - + # Extract config from the parsed data config_data = parsed_data.get("config", {}) # Extract key material if provided @@ -193,47 +194,55 @@ async def client_endpoint(request: Request): # Note: This is a placeholder. The actual implementation would depend on how # the S3EncryptionClient handles key material print(f"Key material provided: {key_material}") - + enable_legacy_wrapping_algorithms = config_data.get("enableLegacyWrappingAlgorithms", False) - + # TODO pull region from ARN kms_client = boto3.client("kms", region_name="us-west-2") - kms_key_id = key_material['kmsKeyId'] - keyring = KmsKeyring(kms_client, kms_key_id=kms_key_id, enable_legacy_wrapping_algorithms=enable_legacy_wrapping_algorithms) + kms_key_id = key_material["kmsKeyId"] + keyring = KmsKeyring( + kms_client, + kms_key_id=kms_key_id, + enable_legacy_wrapping_algorithms=enable_legacy_wrapping_algorithms, + ) wrapped_client = boto3.client("s3") client_config = S3EncryptionClientConfig(keyring) # Create S3EncryptionClientConfig # client_config = S3EncryptionClientConfig( - # enable_legacy_unauthenticated_modes=config_data.get("enableLegacyUnauthenticatedModes", False), - # enable_delayed_authentication_mode=config_data.get("enableDelayedAuthenticationMode", False), - # enable_legacy_wrapping_algorithms=config_data.get("enableLegacyWrappingAlgorithms", False), - # buffer_size=config_data.get("setBufferSize", 0) + # enable_legacy_unauthenticated_modes=config_data.get("enableLegacyUnauthenticatedModes", False), + # enable_delayed_authentication_mode=config_data.get("enableDelayedAuthenticationMode", False), + # enable_legacy_wrapping_algorithms=config_data.get("enableLegacyWrappingAlgorithms", False), + # buffer_size=config_data.get("setBufferSize", 0) # ) - + # Create S3EncryptionClient client = S3EncryptionClient(wrapped_client, client_config) print(f"Created S3EncryptionClient: {client}") - + # Generate a client ID using UUID client_id = str(uuid.uuid4()) - + # Add the client to the client_cache dictionary client_cache[client_id] = client print(f"Added client to cache with ID: {client_id}") - + return {"clientId": client_id} except json.JSONDecodeError as e: print(f"Error parsing JSON: {e}") - return create_generic_server_error("Invalid JSON in request body", status.HTTP_400_BAD_REQUEST) + return create_generic_server_error( + "Invalid JSON in request body", status.HTTP_400_BAD_REQUEST + ) except Exception as e: print(f"Error creating S3EncryptionClient: {e}") return create_s3_encryption_client_error(f"Failed to create client: {str(e)}") + def main(): """ Main function to start the server. """ uvicorn.run(app, host="localhost", port=8081) + if __name__ == "__main__": main() From 4fdce1f30117bc367349f2167855d6bb1800e314 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 18 Aug 2025 11:24:36 -0700 Subject: [PATCH 46/81] ci-fast --- .github/workflows/test.yml | 27 +++++++- test-server/Makefile | 76 +++++++++++++++++++---- test-server/OPTIMIZATION.md | 76 +++++++++++++++++++++++ test-server/README.md | 27 +++++++- test-server/gradle.init | 57 +++++++++++++++++ test-server/java-server/gradle.properties | 8 +++ test-server/java-tests/gradle.properties | 8 +++ 7 files changed, 265 insertions(+), 14 deletions(-) create mode 100644 test-server/OPTIMIZATION.md create mode 100644 test-server/gradle.init diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 670e55da..671096d9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,6 +25,15 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version || '3.11' }} + + # Cache Poetry dependencies + - name: Cache Poetry dependencies + uses: actions/cache@v3 + with: + path: ~/.cache/pypoetry + key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- - name: Install Uv run: pip install uv @@ -35,6 +44,19 @@ jobs: version: latest virtualenvs-create: true virtualenvs-in-project: true + + # Cache Gradle dependencies and build outputs + - name: Cache Gradle packages + uses: actions/cache@v3 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + test-server/java-server/.gradle + test-server/java-tests/.gradle + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- - name: Install dependencies run: make install @@ -54,9 +76,10 @@ jobs: CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} - - name: Run test-server tests - run: cd test-server && make ci + - name: Run test-server tests (optimized) + run: cd test-server && make ci-fast env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} + GRADLE_OPTS: "-Dorg.gradle.daemon=true -Dorg.gradle.parallel=true -Dorg.gradle.caching=true" diff --git a/test-server/Makefile b/test-server/Makefile index 146a53d8..4060b03e 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -1,6 +1,6 @@ # Makefile for S3 Encryption Client Testing -.PHONY: all start-servers run-tests stop-servers clean ci check-env help +.PHONY: all start-servers start-python-server start-java-server run-tests stop-servers clean ci check-env help # Default target all: start-servers run-tests @@ -8,6 +8,9 @@ all: start-servers run-tests # CI target for GitHub Actions ci: start-servers run-tests stop-servers +# Optimized CI target for GitHub Actions +ci-fast: start-servers-parallel run-tests stop-servers + # Start both servers in background with output to stdout (default for debugging) start-servers: @echo "Starting Python server..." @@ -28,7 +31,54 @@ start-servers: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./gradlew run & echo $$! > ../java-server.pid + ./gradlew --build-cache --parallel run & echo $$! > ../java-server.pid + @echo "Waiting for servers to be ready..." + @for i in $$(seq 1 360); do \ + if nc -z localhost 8080 && nc -z localhost 8081; then \ + echo "Ports are open, waiting for servers to initialize..."; \ + sleep 5; \ + echo "Both servers are ready!"; \ + break; \ + fi; \ + if [ $$i -eq 360 ]; then \ + echo "Timeout waiting for servers to start"; \ + exit 1; \ + fi; \ + echo "Waiting for servers to start ($$i/360)..."; \ + sleep 1; \ + done + +# Start Python server in background +start-python-server: + @echo "Starting Python server..." + cd python-server && \ + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + poetry install --no-interaction && \ + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + poetry run python src/main.py & echo $$! > ../python-server.pid + @echo "Python server starting..." + +# Start Java server in background +start-java-server: + @echo "Starting Java server..." + cd java-server && \ + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + ./gradlew --build-cache --parallel run & echo $$! > ../java-server.pid + @echo "Java server starting..." + +# Start both servers in parallel +start-servers-parallel: + @echo "Starting servers in parallel..." + @$(MAKE) -j2 start-python-server start-java-server @echo "Waiting for servers to be ready..." @for i in $$(seq 1 360); do \ if nc -z localhost 8080 && nc -z localhost 8081; then \ @@ -56,7 +106,7 @@ run-tests: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./gradlew integ + ./gradlew --build-cache --parallel integ @echo "Tests completed successfully" # Stop the servers @@ -81,14 +131,18 @@ clean: stop-servers # Help target help: @echo "Available targets:" - @echo " all : Start servers and run tests (default, output to stdout)" - @echo " ci : Run in CI mode (start servers, run tests, stop servers)" - @echo " start-servers: Start Python and Java servers (output to stdout)" - @echo " run-tests : Run Java tests" - @echo " stop-servers : Stop running servers" - @echo " clean : Stop servers and clean up logs" - @echo " check-env : Check if required environment variables are set" - @echo " help : Show this help message" + @echo " all : Start servers and run tests (default, output to stdout)" + @echo " ci : Run in CI mode (start servers, run tests, stop servers)" + @echo " ci-fast : Run in optimized CI mode (start servers in parallel, run tests, stop servers)" + @echo " start-servers : Start Python and Java servers sequentially (output to stdout)" + @echo " start-servers-parallel: Start Python and Java servers in parallel (output to stdout)" + @echo " start-python-server: Start only the Python server" + @echo " start-java-server : Start only the Java server" + @echo " run-tests : Run Java tests" + @echo " stop-servers : Stop running servers" + @echo " clean : Stop servers and clean up logs" + @echo " check-env : Check if required environment variables are set" + @echo " help : Show this help message" # Check if required environment variables are set check-env: diff --git a/test-server/OPTIMIZATION.md b/test-server/OPTIMIZATION.md new file mode 100644 index 00000000..a50e70a7 --- /dev/null +++ b/test-server/OPTIMIZATION.md @@ -0,0 +1,76 @@ +# Test Server Performance Optimizations + +This document describes the performance optimizations implemented to speed up the test-server CI process. + +## Overview + +The test-server CI process involves starting both Python and Java servers, then running Java tests against them. The original implementation was taking over 5 minutes to run, with most of the time spent on Gradle/Java setup rather than the actual tests. + +## Optimizations Implemented + +### 1. Parallel Server Startup + +- Added a new `start-servers-parallel` target in the Makefile that starts both Python and Java servers concurrently +- Created a new `ci-fast` target that uses parallel server startup + +### 2. Gradle Performance Optimizations + +- Added Gradle build caching +- Enabled parallel execution of Gradle tasks +- Configured the Gradle daemon for faster startup +- Optimized JVM memory settings +- Added incremental compilation +- Configured parallel test execution + +### 3. CI Workflow Optimizations + +- Added caching for Gradle dependencies and build outputs +- Added caching for Poetry dependencies +- Set environment variables to ensure Gradle optimizations are used + +## Configuration Files + +The following files were modified or created: + +1. `test-server/Makefile`: Added new targets for parallel execution +2. `.github/workflows/test.yml`: Added caching and updated to use the optimized CI target +3. `test-server/java-server/gradle.properties` and `test-server/java-tests/gradle.properties`: Added performance settings +4. `test-server/gradle.init`: Added global Gradle settings for all projects + +## Usage + +### Local Development + +For local development and testing, you can use the optimized targets: + +```bash +# Run the optimized CI process +cd test-server && make ci-fast + +# Start servers in parallel +cd test-server && make start-servers-parallel + +# Run tests with optimized Gradle settings +cd test-server/java-tests && ./gradlew --build-cache --parallel integ +``` + +### CI Environment + +The GitHub Actions workflow has been updated to use the optimized CI process automatically. + +## Performance Impact + +The optimizations are expected to significantly reduce the CI execution time by: + +1. Running server startup in parallel (saves time equal to the slower of the two servers) +2. Caching Gradle and Poetry dependencies (saves download and resolution time) +3. Optimizing Gradle execution (reduces build time) +4. Enabling incremental compilation (reduces compilation time on subsequent runs) + +## Troubleshooting + +If you encounter issues with the optimized CI process: + +1. Try running the original `ci` target: `make ci` +2. Check Gradle daemon logs: `cat ~/.gradle/daemon/*/daemon-*.out.log` +3. Disable specific optimizations by modifying the relevant configuration files diff --git a/test-server/README.md b/test-server/README.md index 06ce12ed..a030f6e3 100644 --- a/test-server/README.md +++ b/test-server/README.md @@ -25,9 +25,21 @@ make # Run in CI mode (start servers, run tests, stop servers) make ci -# Start Python and Java servers +# Run in optimized CI mode (start servers in parallel, run tests, stop servers) +make ci-fast + +# Start Python and Java servers sequentially make start-servers +# Start Python and Java servers in parallel +make start-servers-parallel + +# Start only the Python server +make start-python-server + +# Start only the Java server +make start-java-server + # Run Java tests make run-tests @@ -42,3 +54,16 @@ make help ``` The `ci` target is specifically designed for GitHub Actions workflows, ensuring that servers are properly started, tests are run, and resources are cleaned up afterward. + +The `ci-fast` target is an optimized version that starts servers in parallel and uses various performance optimizations to speed up the CI process. + +## Performance Optimizations + +Performance optimizations have been implemented to speed up the test-server CI process, which was previously taking over 5 minutes to run. These optimizations include: + +- Parallel server startup +- Gradle build caching and parallel execution +- Dependency caching in CI +- JVM optimizations + +For detailed information about the optimizations, see [OPTIMIZATION.md](./OPTIMIZATION.md). diff --git a/test-server/gradle.init b/test-server/gradle.init new file mode 100644 index 00000000..7091c254 --- /dev/null +++ b/test-server/gradle.init @@ -0,0 +1,57 @@ +// Global initialization script for Gradle +// This applies to all Gradle builds in subdirectories + +gradle.projectsLoaded { + rootProject.allprojects { + buildscript { + // Configure build script classpath repositories + repositories { + mavenLocal() + mavenCentral() + gradlePluginPortal() + } + } + } +} + +// Apply common settings to all projects +allprojects { + // Configure project repositories + repositories { + mavenLocal() + mavenCentral() + } + + // Configure tasks + tasks.withType(JavaCompile) { + options.incremental = true + options.fork = true + } + + // Configure test tasks + tasks.withType(Test) { + maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 + testLogging { + events "passed", "skipped", "failed" + exceptionFormat = 'full' + } + } +} + +// Initialize Gradle with optimized settings if not already set +startParameter.with { + if (!systemPropertiesDefined('org.gradle.parallel')) { + systemProperties['org.gradle.parallel'] = 'true' + } + if (!systemPropertiesDefined('org.gradle.caching')) { + systemProperties['org.gradle.caching'] = 'true' + } + if (!systemPropertiesDefined('org.gradle.daemon')) { + systemProperties['org.gradle.daemon'] = 'true' + } +} + +// Helper method to check if a system property is defined +boolean systemPropertiesDefined(String property) { + return System.properties.containsKey(property) || startParameter.systemPropertiesArgs.containsKey(property) +} diff --git a/test-server/java-server/gradle.properties b/test-server/java-server/gradle.properties index 0af8556d..08afce82 100644 --- a/test-server/java-server/gradle.properties +++ b/test-server/java-server/gradle.properties @@ -1,3 +1,11 @@ +# Smithy versions smithyJavaVersion=[0,1] smithyGradleVersion=1.1.0 smithyVersion=[1,2] + +# Performance optimization settings +org.gradle.parallel=true +org.gradle.caching=true +org.gradle.daemon=true +org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError +org.gradle.workers.max=4 diff --git a/test-server/java-tests/gradle.properties b/test-server/java-tests/gradle.properties index 0af8556d..08afce82 100644 --- a/test-server/java-tests/gradle.properties +++ b/test-server/java-tests/gradle.properties @@ -1,3 +1,11 @@ +# Smithy versions smithyJavaVersion=[0,1] smithyGradleVersion=1.1.0 smithyVersion=[1,2] + +# Performance optimization settings +org.gradle.parallel=true +org.gradle.caching=true +org.gradle.daemon=true +org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError +org.gradle.workers.max=4 From 035dff002744da86e2df21e2820c4ac811bf834b Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 18 Aug 2025 11:44:09 -0700 Subject: [PATCH 47/81] remove print --- .../s3/CreateClientOperationImpl.java | 1 - .../encryption/s3/GetObjectOperationImpl.java | 2 -- .../encryption/s3/PutObjectOperationImpl.java | 3 --- .../amazon/encryption/s3/RoundTripTests.java | 2 -- test-server/python-server/src/main.py | 25 ------------------- 5 files changed, 33 deletions(-) diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java b/test-server/java-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java index f798eb5d..1a609fed 100644 --- a/test-server/java-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java +++ b/test-server/java-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java @@ -53,7 +53,6 @@ private boolean onlyOneNonNull(Object... values) { @Override public CreateClientOutput createClient(CreateClientInput input, RequestContext context) { - System.out.println("createClient called!"); try { KeyMaterial key = input.config().keyMaterial(); if (!onlyOneNonNull(key.aesKey(), key.kmsKeyId(), key.rsaKey())) { diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java b/test-server/java-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java index 95529511..7d2bc2ea 100644 --- a/test-server/java-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java +++ b/test-server/java-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java @@ -30,7 +30,6 @@ public GetObjectOperationImpl(Map clientCache) { @Override public GetObjectOutput getObject(GetObjectInput input, RequestContext context) { try { - System.out.println("Getting object with ClientId: " + input.clientID()); S3Client s3Client = clientCache_.get(input.clientID()); Map ecMap = metadataListToMap(input.metadata()); @@ -47,7 +46,6 @@ public GetObjectOutput getObject(GetObjectInput input, RequestContext context) { .body(bb) .metadata(mdAsList) .build(); - System.out.println("returning"); return output; } catch (S3EncryptionClientException s3EncryptionClientException) { // Modeled exceptions MUST be returned as such diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java b/test-server/java-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java index c7bbbbe2..a281a0b8 100644 --- a/test-server/java-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java +++ b/test-server/java-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java @@ -29,8 +29,6 @@ public PutObjectOperationImpl(Map clientCache) { @Override public PutObjectOutput putObject(PutObjectInput input, RequestContext context) { try { - System.out.println("Putting object with ClientId: " + input.clientID()); - System.out.println("putting " + input.key() + " in bucket " + input.bucket() + " with content: " + input.body()); final Map metadata = metadataListToMap(input.metadata()); S3Client s3Client = clientCache_.get(input.clientID()); s3Client.putObject(builder -> builder @@ -39,7 +37,6 @@ public PutObjectOutput putObject(PutObjectInput input, RequestContext context) { .overrideConfiguration(withAdditionalConfiguration(metadata)), RequestBody.fromByteBuffer(input.body()) ); - System.out.println("Success!"); // The real S3 doesn't provide bucket/key/metadata, so Test doesn't need to either, but we do anyway return PutObjectOutput.builder() .bucket(input.bucket()) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index b7851e11..7fd96019 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -206,7 +206,6 @@ public void crossLanguageTestKms(LanguageServerTarget encLang, LanguageServerTar .build()); if (!input.equals(StandardCharsets.UTF_8.decode(output.body()).toString())) { - System.out.println(String.format("Encryption in %s failed to decrpyt in %s!", encLang, decLang)); fail(String.format("Encryption in %s failed to decrpyt in %s!", encLang, decLang)); } } @@ -251,7 +250,6 @@ public void crossLanguageTestKmsWithEncCtx(LanguageServerTarget encLang, Languag .build()); if (!input.equals(StandardCharsets.UTF_8.decode(output.body()).toString())) { - System.out.println(String.format("Encryption in %s failed to decrpyt in %s!", encLang, decLang)); fail(String.format("Encryption in %s failed to decrpyt in %s!", encLang, decLang)); } } diff --git a/test-server/python-server/src/main.py b/test-server/python-server/src/main.py index 05405d69..13ddbf5e 100755 --- a/test-server/python-server/src/main.py +++ b/test-server/python-server/src/main.py @@ -75,8 +75,6 @@ async def put_object(bucket: str, key: str, request: Request): """ client_id = request.headers.get("ClientID") body = await request.body() - print(f"PUT object request - Bucket: {bucket}, Key: {key}") - print(f"ClientID from header: {client_id}") if not client_id: return create_generic_server_error( @@ -99,8 +97,6 @@ async def put_object(bucket: str, key: str, request: Request): **{"Bucket": bucket, "Key": key, "Body": body, "EncryptionContext": enc_ctx} ) - print(f"PutObject response: {response}") - # Return the appropriate response return { "bucket": bucket, @@ -108,7 +104,6 @@ async def put_object(bucket: str, key: str, request: Request): "metadata": metadata if isinstance(metadata, list) else [], } except Exception as e: - print(f"Error making PutObject request: {e}") return create_s3_encryption_client_error(f"Failed to put object: {str(e)}") @@ -119,8 +114,6 @@ async def get_object(bucket: str, key: str, request: Request): to make a GetObject request to S3. """ client_id = request.headers.get("ClientID") - print(f"GET object request - Bucket: {bucket}, Key: {key}") - print(f"ClientID from header: {client_id}") if not client_id: return create_generic_server_error( @@ -139,16 +132,11 @@ async def get_object(bucket: str, key: str, request: Request): try: # Use the client to make a GetObject request to S3 - print("making Get for " + key) response = client.get_object(**{"Bucket": bucket, "Key": key, "EncryptionContext": enc_ctx}) - print(f"GetObject response: {response}") - # Extract the body and metadata from the response body = response.get("Body").read() if response.get("Body") else b"" - # print(f"body:" + body) metadata = response.get("Metadata", []) - print(f"md: {metadata}") # Convert metadata dictionary to a list of key-value pairs if it's a dict if isinstance(metadata, dict): @@ -160,15 +148,12 @@ async def get_object(bucket: str, key: str, request: Request): # Convert metadata_list to a comma-separated string metadata_str = ",".join(metadata_list) if metadata_list else "" headers = {"Content-Metadata": metadata_str} - print(f"headers: {headers}") # Return the body as the response payload return Response(content=body, headers=headers) except S3EncryptionClientError as ex: - print(f"Modeled Error making GetObject request: {ex}") return create_s3_encryption_client_error(str(ex)) except Exception as e: - print(f"Generic Error making GetObject request: {e}") return create_generic_server_error(e) @@ -178,22 +163,16 @@ async def client_endpoint(request: Request): Handle POST requests to /client by creating an S3EncryptionClient. """ body = await request.body() - print(f"Received client request with body: {body}") # Parse the bytes object as JSON try: # Decode bytes to string and parse as JSON parsed_data = json.loads(body.decode("utf-8")) - print(f"Parsed JSON data: {parsed_data}") # Extract config from the parsed data config_data = parsed_data.get("config", {}) # Extract key material if provided key_material = config_data.get("keyMaterial", {}) - if key_material: - # Note: This is a placeholder. The actual implementation would depend on how - # the S3EncryptionClient handles key material - print(f"Key material provided: {key_material}") enable_legacy_wrapping_algorithms = config_data.get("enableLegacyWrappingAlgorithms", False) @@ -217,23 +196,19 @@ async def client_endpoint(request: Request): # Create S3EncryptionClient client = S3EncryptionClient(wrapped_client, client_config) - print(f"Created S3EncryptionClient: {client}") # Generate a client ID using UUID client_id = str(uuid.uuid4()) # Add the client to the client_cache dictionary client_cache[client_id] = client - print(f"Added client to cache with ID: {client_id}") return {"clientId": client_id} except json.JSONDecodeError as e: - print(f"Error parsing JSON: {e}") return create_generic_server_error( "Invalid JSON in request body", status.HTTP_400_BAD_REQUEST ) except Exception as e: - print(f"Error creating S3EncryptionClient: {e}") return create_s3_encryption_client_error(f"Failed to create client: {str(e)}") From abe71148639b4dc236804dd1bfae271739f37ab0 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 18 Aug 2025 14:57:02 -0700 Subject: [PATCH 48/81] default to fast ci --- .github/workflows/test.yml | 4 ++-- test-server/Makefile | 45 ++----------------------------------- test-server/OPTIMIZATION.md | 18 +++++++-------- test-server/README.md | 14 +++--------- 4 files changed, 16 insertions(+), 65 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 671096d9..38674c27 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -76,8 +76,8 @@ jobs: CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} - - name: Run test-server tests (optimized) - run: cd test-server && make ci-fast + - name: Run test-server tests + run: cd test-server && make ci env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} diff --git a/test-server/Makefile b/test-server/Makefile index 4060b03e..41db943f 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -8,45 +8,6 @@ all: start-servers run-tests # CI target for GitHub Actions ci: start-servers run-tests stop-servers -# Optimized CI target for GitHub Actions -ci-fast: start-servers-parallel run-tests stop-servers - -# Start both servers in background with output to stdout (default for debugging) -start-servers: - @echo "Starting Python server..." - cd python-server && \ - AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ - AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ - AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ - AWS_REGION="us-west-2" \ - poetry install --no-interaction && \ - AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ - AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ - AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ - AWS_REGION="us-west-2" \ - poetry run python src/main.py & echo $$! > ../python-server.pid - @echo "Starting Java server..." - cd java-server && \ - AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ - AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ - AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ - AWS_REGION="us-west-2" \ - ./gradlew --build-cache --parallel run & echo $$! > ../java-server.pid - @echo "Waiting for servers to be ready..." - @for i in $$(seq 1 360); do \ - if nc -z localhost 8080 && nc -z localhost 8081; then \ - echo "Ports are open, waiting for servers to initialize..."; \ - sleep 5; \ - echo "Both servers are ready!"; \ - break; \ - fi; \ - if [ $$i -eq 360 ]; then \ - echo "Timeout waiting for servers to start"; \ - exit 1; \ - fi; \ - echo "Waiting for servers to start ($$i/360)..."; \ - sleep 1; \ - done # Start Python server in background start-python-server: @@ -76,7 +37,7 @@ start-java-server: @echo "Java server starting..." # Start both servers in parallel -start-servers-parallel: +start-servers: @echo "Starting servers in parallel..." @$(MAKE) -j2 start-python-server start-java-server @echo "Waiting for servers to be ready..." @@ -133,9 +94,7 @@ help: @echo "Available targets:" @echo " all : Start servers and run tests (default, output to stdout)" @echo " ci : Run in CI mode (start servers, run tests, stop servers)" - @echo " ci-fast : Run in optimized CI mode (start servers in parallel, run tests, stop servers)" - @echo " start-servers : Start Python and Java servers sequentially (output to stdout)" - @echo " start-servers-parallel: Start Python and Java servers in parallel (output to stdout)" + @echo " start-servers : Start Python and Java servers in parallel (output to stdout)" @echo " start-python-server: Start only the Python server" @echo " start-java-server : Start only the Java server" @echo " run-tests : Run Java tests" diff --git a/test-server/OPTIMIZATION.md b/test-server/OPTIMIZATION.md index a50e70a7..1d2237fb 100644 --- a/test-server/OPTIMIZATION.md +++ b/test-server/OPTIMIZATION.md @@ -10,8 +10,8 @@ The test-server CI process involves starting both Python and Java servers, then ### 1. Parallel Server Startup -- Added a new `start-servers-parallel` target in the Makefile that starts both Python and Java servers concurrently -- Created a new `ci-fast` target that uses parallel server startup +- Updated the `start-servers` target in the Makefile to start both Python and Java servers concurrently +- Updated the `ci` target to use parallel server startup ### 2. Gradle Performance Optimizations @@ -44,11 +44,11 @@ The following files were modified or created: For local development and testing, you can use the optimized targets: ```bash -# Run the optimized CI process -cd test-server && make ci-fast +# Run the CI process (now optimized by default) +cd test-server && make ci # Start servers in parallel -cd test-server && make start-servers-parallel +cd test-server && make start-servers # Run tests with optimized Gradle settings cd test-server/java-tests && ./gradlew --build-cache --parallel integ @@ -69,8 +69,8 @@ The optimizations are expected to significantly reduce the CI execution time by: ## Troubleshooting -If you encounter issues with the optimized CI process: +If you encounter issues with the CI process: -1. Try running the original `ci` target: `make ci` -2. Check Gradle daemon logs: `cat ~/.gradle/daemon/*/daemon-*.out.log` -3. Disable specific optimizations by modifying the relevant configuration files +1. Check Gradle daemon logs: `cat ~/.gradle/daemon/*/daemon-*.out.log` +2. Disable specific optimizations by modifying the relevant configuration files +3. Try running the servers sequentially by modifying the Makefile diff --git a/test-server/README.md b/test-server/README.md index a030f6e3..ca8ee6a7 100644 --- a/test-server/README.md +++ b/test-server/README.md @@ -22,17 +22,11 @@ A Makefile is provided to simplify running the servers and tests. The Makefile h # Start servers and run tests (default) make -# Run in CI mode (start servers, run tests, stop servers) +# Run in CI mode (start servers in parallel, run tests, stop servers) make ci -# Run in optimized CI mode (start servers in parallel, run tests, stop servers) -make ci-fast - -# Start Python and Java servers sequentially -make start-servers - # Start Python and Java servers in parallel -make start-servers-parallel +make start-servers # Start only the Python server make start-python-server @@ -53,9 +47,7 @@ make clean make help ``` -The `ci` target is specifically designed for GitHub Actions workflows, ensuring that servers are properly started, tests are run, and resources are cleaned up afterward. - -The `ci-fast` target is an optimized version that starts servers in parallel and uses various performance optimizations to speed up the CI process. +The `ci` target is specifically designed for GitHub Actions workflows, ensuring that servers are properly started in parallel, tests are run, and resources are cleaned up afterward. ## Performance Optimizations From a49a0189f9cd8832509296da5aaf9280b9033876 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Thu, 28 Aug 2025 19:04:06 -0700 Subject: [PATCH 49/81] fix smithy breaking changes, poetry to uv --- .github/workflows/test.yml | 17 +- test-server/Makefile | 11 +- test-server/OPTIMIZATION.md | 4 +- test-server/java-server/settings.gradle.kts | 2 +- .../s3/CreateClientOperationImpl.java | 26 +- .../encryption/s3/GetObjectOperationImpl.java | 8 +- .../encryption/s3/PutObjectOperationImpl.java | 16 +- .../amazon/encryption/s3/RoundTripTests.java | 26 +- test-server/python-server/.gitignore | 3 +- test-server/python-server/README.md | 22 +- test-server/python-server/poetry.lock | 865 ++++++++++++++++++ test-server/python-server/pyproject.toml | 32 +- 12 files changed, 947 insertions(+), 85 deletions(-) create mode 100644 test-server/python-server/poetry.lock diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 38674c27..f8025246 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,24 +26,17 @@ jobs: with: python-version: ${{ inputs.python-version || '3.11' }} - # Cache Poetry dependencies - - name: Cache Poetry dependencies + # Cache uv dependencies + - name: Cache uv dependencies uses: actions/cache@v3 with: - path: ~/.cache/pypoetry - key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} + path: ~/.cache/uv + key: ${{ runner.os }}-uv-${{ hashFiles('**/pyproject.toml') }} restore-keys: | - ${{ runner.os }}-poetry- + ${{ runner.os }}-uv- - name: Install Uv run: pip install uv - - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - version: latest - virtualenvs-create: true - virtualenvs-in-project: true # Cache Gradle dependencies and build outputs - name: Cache Gradle packages diff --git a/test-server/Makefile b/test-server/Makefile index 41db943f..afbe97b5 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -13,16 +13,15 @@ ci: start-servers run-tests stop-servers start-python-server: @echo "Starting Python server..." cd python-server && \ + python -m venv .venv && \ + .venv/bin/python -m ensurepip && \ + .venv/bin/python -m pip install -e . && \ + .venv/bin/python -m pip install -e ../.. && \ AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - poetry install --no-interaction && \ - AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ - AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ - AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ - AWS_REGION="us-west-2" \ - poetry run python src/main.py & echo $$! > ../python-server.pid + .venv/bin/python src/main.py & echo $$! > ../python-server.pid @echo "Python server starting..." # Start Java server in background diff --git a/test-server/OPTIMIZATION.md b/test-server/OPTIMIZATION.md index 1d2237fb..7774cf78 100644 --- a/test-server/OPTIMIZATION.md +++ b/test-server/OPTIMIZATION.md @@ -25,7 +25,7 @@ The test-server CI process involves starting both Python and Java servers, then ### 3. CI Workflow Optimizations - Added caching for Gradle dependencies and build outputs -- Added caching for Poetry dependencies +- Added caching for uv dependencies - Set environment variables to ensure Gradle optimizations are used ## Configuration Files @@ -63,7 +63,7 @@ The GitHub Actions workflow has been updated to use the optimized CI process aut The optimizations are expected to significantly reduce the CI execution time by: 1. Running server startup in parallel (saves time equal to the slower of the two servers) -2. Caching Gradle and Poetry dependencies (saves download and resolution time) +2. Caching Gradle and uv dependencies (saves download and resolution time) 3. Optimizing Gradle execution (reduces build time) 4. Enabling incremental compilation (reduces compilation time on subsequent runs) diff --git a/test-server/java-server/settings.gradle.kts b/test-server/java-server/settings.gradle.kts index c608c023..e7c41714 100644 --- a/test-server/java-server/settings.gradle.kts +++ b/test-server/java-server/settings.gradle.kts @@ -16,4 +16,4 @@ pluginManagement { } } -rootProject.name = "BasicSmithyJavaServer" +rootProject.name = "S3ECJavaTestServer" diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java b/test-server/java-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java index 1a609fed..d992c435 100644 --- a/test-server/java-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java +++ b/test-server/java-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java @@ -54,36 +54,36 @@ private boolean onlyOneNonNull(Object... values) { @Override public CreateClientOutput createClient(CreateClientInput input, RequestContext context) { try { - KeyMaterial key = input.config().keyMaterial(); - if (!onlyOneNonNull(key.aesKey(), key.kmsKeyId(), key.rsaKey())) { + KeyMaterial key = input.getConfig().getKeyMaterial(); + if (!onlyOneNonNull(key.getAesKey(), key.getKmsKeyId(), key.getRsaKey())) { throw new RuntimeException("KeyMaterial must be only one, non-null input!"); } Keyring keyring; - if (key.aesKey() != null) { - byte[] keyBytes = new byte[key.aesKey().remaining()]; - key.aesKey().get(keyBytes); + if (key.getAesKey() != null) { + byte[] keyBytes = new byte[key.getAesKey().remaining()]; + key.getAesKey().get(keyBytes); keyring = AesKeyring.builder() .wrappingKey(new SecretKeySpec(keyBytes, "AES")) - .enableLegacyWrappingAlgorithms(input.config().enableLegacyWrappingAlgorithms()) + .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) .build(); - } else if (key.rsaKey() != null) { + } else if (key.getRsaKey() != null) { try { - byte[] keyBytes = new byte[key.rsaKey().remaining()]; - key.rsaKey().get(keyBytes); + byte[] keyBytes = new byte[key.getRsaKey().remaining()]; + key.getRsaKey().get(keyBytes); PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(keyBytes); KeyFactory keyFactory = KeyFactory.getInstance("RSA"); keyring = RsaKeyring.builder() - .enableLegacyWrappingAlgorithms(input.config().enableLegacyWrappingAlgorithms()) + .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) .wrappingKeyPair(PartialRsaKeyPair.builder() .privateKey(keyFactory.generatePrivate(keySpec)).build()) .build(); } catch (NoSuchAlgorithmException | InvalidKeySpecException nse) { throw new RuntimeException(nse); } - } else if (key.kmsKeyId() != null) { + } else if (key.getKmsKeyId() != null) { keyring = KmsKeyring.builder() - .enableLegacyWrappingAlgorithms(input.config().enableLegacyWrappingAlgorithms()) - .wrappingKeyId(key.kmsKeyId()) + .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) + .wrappingKeyId(key.getKmsKeyId()) .build(); } else { throw new RuntimeException("No KeyMaterial found!"); diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java b/test-server/java-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java index 7d2bc2ea..e7c5493f 100644 --- a/test-server/java-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java +++ b/test-server/java-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java @@ -30,13 +30,13 @@ public GetObjectOperationImpl(Map clientCache) { @Override public GetObjectOutput getObject(GetObjectInput input, RequestContext context) { try { - S3Client s3Client = clientCache_.get(input.clientID()); - Map ecMap = metadataListToMap(input.metadata()); + S3Client s3Client = clientCache_.get(input.getClientID()); + Map ecMap = metadataListToMap(input.getMetadata()); try { ResponseBytes resp = s3Client.getObjectAsBytes(builder -> builder - .bucket(input.bucket()) - .key(input.key()) + .bucket(input.getBucket()) + .key(input.getKey()) .overrideConfiguration(withAdditionalConfiguration(ecMap))); List mdAsList = metadataMapToList(resp.response().metadata()); diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java b/test-server/java-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java index a281a0b8..4c772673 100644 --- a/test-server/java-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java +++ b/test-server/java-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java @@ -29,19 +29,19 @@ public PutObjectOperationImpl(Map clientCache) { @Override public PutObjectOutput putObject(PutObjectInput input, RequestContext context) { try { - final Map metadata = metadataListToMap(input.metadata()); - S3Client s3Client = clientCache_.get(input.clientID()); + final Map metadata = metadataListToMap(input.getMetadata()); + S3Client s3Client = clientCache_.get(input.getClientID()); s3Client.putObject(builder -> builder - .bucket(input.bucket()) - .key(input.key()) + .bucket(input.getBucket()) + .key(input.getKey()) .overrideConfiguration(withAdditionalConfiguration(metadata)), - RequestBody.fromByteBuffer(input.body()) + RequestBody.fromByteBuffer(input.getBody()) ); // The real S3 doesn't provide bucket/key/metadata, so Test doesn't need to either, but we do anyway return PutObjectOutput.builder() - .bucket(input.bucket()) - .key(input.key()) - .metadata(input.metadata()) + .bucket(input.getBucket()) + .key(input.getKey()) + .metadata(input.getMetadata()) .build(); } catch (Exception e) { StringWriter sw = new StringWriter(); diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index 7fd96019..ff50fc85 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -186,7 +186,7 @@ public void crossLanguageTestKms(LanguageServerTarget encLang, LanguageServerTar .config(S3ECConfig.builder() .keyMaterial(kmsKeyArn).build()) .build()); - String encS3ECId = encClientOutput.clientId(); + String encS3ECId = encClientOutput.getClientId(); encClient.putObject(PutObjectInput.builder() .clientID(encS3ECId) .key(objectKey) @@ -198,14 +198,14 @@ public void crossLanguageTestKms(LanguageServerTarget encLang, LanguageServerTar .config(S3ECConfig.builder() .keyMaterial(kmsKeyArn).build()) .build()); - String decS3ECId = decClientOutput.clientId(); + String decS3ECId = decClientOutput.getClientId(); GetObjectOutput output = decClient.getObject(GetObjectInput.builder() .clientID(decS3ECId) .bucket(BUCKET) .key(objectKey) .build()); - if (!input.equals(StandardCharsets.UTF_8.decode(output.body()).toString())) { + if (!input.equals(StandardCharsets.UTF_8.decode(output.getBody()).toString())) { fail(String.format("Encryption in %s failed to decrpyt in %s!", encLang, decLang)); } } @@ -227,7 +227,7 @@ public void crossLanguageTestKmsWithEncCtx(LanguageServerTarget encLang, Languag .config(S3ECConfig.builder() .keyMaterial(kmsKeyArn).build()) .build()); - String encS3ECId = encClientOutput.clientId(); + String encS3ECId = encClientOutput.getClientId(); encClient.putObject(PutObjectInput.builder() .clientID(encS3ECId) @@ -241,7 +241,7 @@ public void crossLanguageTestKmsWithEncCtx(LanguageServerTarget encLang, Languag .config(S3ECConfig.builder() .keyMaterial(kmsKeyArn).build()) .build()); - String decS3ECId = decClientOutput.clientId(); + String decS3ECId = decClientOutput.getClientId(); GetObjectOutput output = decClient.getObject(GetObjectInput.builder() .clientID(decS3ECId) .bucket(BUCKET) @@ -249,7 +249,7 @@ public void crossLanguageTestKmsWithEncCtx(LanguageServerTarget encLang, Languag .metadata(mdAsList) .build()); - if (!input.equals(StandardCharsets.UTF_8.decode(output.body()).toString())) { + if (!input.equals(StandardCharsets.UTF_8.decode(output.getBody()).toString())) { fail(String.format("Encryption in %s failed to decrpyt in %s!", encLang, decLang)); } } @@ -271,7 +271,7 @@ public void crossLanguageTestKmsWithEncCtxMismatchFails(LanguageServerTarget enc .config(S3ECConfig.builder() .keyMaterial(kmsKeyArn).build()) .build()); - String encS3ECId = encClientOutput.clientId(); + String encS3ECId = encClientOutput.getClientId(); encClient.putObject(PutObjectInput.builder() .clientID(encS3ECId) @@ -285,7 +285,7 @@ public void crossLanguageTestKmsWithEncCtxMismatchFails(LanguageServerTarget enc .config(S3ECConfig.builder() .keyMaterial(kmsKeyArn).build()) .build()); - String decS3ECId = decClientOutput.clientId(); + String decS3ECId = decClientOutput.getClientId(); try { decClient.getObject(GetObjectInput.builder() .clientID(decS3ECId) @@ -313,7 +313,7 @@ public void kmsV1Legacy(String language) { .keyMaterial(kmsKeyArn) .build()) .build()); - String s3ECId = output1.clientId(); + String s3ECId = output1.getClientId(); // Create the object using the old client // V1 Client @@ -337,7 +337,7 @@ public void kmsV1Legacy(String language) { .key(objectKey) .build()); - assertEquals(input, new String(output.body().array())); + assertEquals(input, new String(output.getBody().array())); } @ParameterizedTest(name = "{displayName} for Encrypt: Java, Decrypt: {0}") @@ -355,7 +355,7 @@ public void kmsV1LegacyWithEncCtx(String language) { .keyMaterial(kmsKeyArn) .build()) .build()); - String s3ECId = output1.clientId(); + String s3ECId = output1.getClientId(); // Create the object using the old client // V1 Client @@ -386,7 +386,7 @@ public void kmsV1LegacyWithEncCtx(String language) { .metadata(metadataMapToList(encCtx)) .build()); - assertEquals(input, new String(output.body().array())); + assertEquals(input, new String(output.getBody().array())); } @ParameterizedTest(name = "{displayName} for Encrypt: Java, Decrypt: {0}") @@ -404,7 +404,7 @@ public void kmsV1LegacyFailsWhenLegacyDisabled(String language) { .keyMaterial(kmsKeyArn) .build()) .build()); - String s3ECId = output1.clientId(); + String s3ECId = output1.getClientId(); // Create the object using the old client // V1 Client diff --git a/test-server/python-server/.gitignore b/test-server/python-server/.gitignore index 1089c7d2..da84c314 100644 --- a/test-server/python-server/.gitignore +++ b/test-server/python-server/.gitignore @@ -20,8 +20,7 @@ wheels/ .installed.cfg *.egg -# Poetry -poetry.lock +# Python virtual environment .venv/ venv/ diff --git a/test-server/python-server/README.md b/test-server/python-server/README.md index 619e030a..f9e4fd5b 100644 --- a/test-server/python-server/README.md +++ b/test-server/python-server/README.md @@ -4,28 +4,31 @@ A FastAPI-based Python server implementation. ## Setup -1. Install Poetry (if not already installed): +1. Install uv (if not already installed): ```bash -curl -sSL https://install.python-poetry.org | python3 - +pip install uv ``` -2. Install dependencies: +2. Create a virtual environment and install dependencies: ```bash -poetry install +uv venv +source .venv/bin/activate +uv pip install -e . +uv pip install -e ../.. ``` ## Development - Source code is in the `src` directory - Tests are in the `tests` directory -- Use `poetry shell` to activate the virtual environment -- Use `poetry add {package}` to add new dependencies -- Use `poetry add -D {package}` to add new development dependencies +- Use `source .venv/bin/activate` to activate the virtual environment +- Use `uv pip install {package}` to add new dependencies +- Use `uv pip install {package} --dev` to add new development dependencies ## Running the Server ```bash -poetry run python src/main.py +.venv/bin/python src/main.py ``` The server will start on `http://localhost:8080` with the following endpoints: @@ -39,4 +42,5 @@ The server will start on `http://localhost:8080` with the following endpoints: ## Running Tests ```bash -poetry run pytest +.venv/bin/python -m pytest +``` diff --git a/test-server/python-server/poetry.lock b/test-server/python-server/poetry.lock new file mode 100644 index 00000000..b156c48c --- /dev/null +++ b/test-server/python-server/poetry.lock @@ -0,0 +1,865 @@ +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. + +[[package]] +name = "amazon-s3-encryption-client-python" +version = "0.1.0" +description = "This library provides an S3 client that supports client-side encryption." +optional = false +python-versions = "^3.11" +files = [] +develop = true + +[package.dependencies] +attrs = "^25.1.0" +aws-cryptographic-material-providers = "^1.7.4" +boto3 = "^1.37.2" +cryptography = "^43.0.1" +pytest = "^8.4.1" + +[package.source] +type = "directory" +url = "../../amazon-s3-encryption-client-python" + +[[package]] +name = "annotated-types" +version = "0.7.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, +] + +[[package]] +name = "anyio" +version = "4.9.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.9" +files = [ + {file = "anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c"}, + {file = "anyio-4.9.0.tar.gz", hash = "sha256:673c0c244e15788651a4ff38710fea9675823028a6f08a5eda409e0c9840a028"}, +] + +[package.dependencies] +idna = ">=2.8" +sniffio = ">=1.1" +typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} + +[package.extras] +doc = ["Sphinx (>=8.2,<9.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"] +test = ["anyio[trio]", "blockbuster (>=1.5.23)", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21)"] +trio = ["trio (>=0.26.1)"] + +[[package]] +name = "attrs" +version = "25.3.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.8" +files = [ + {file = "attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3"}, + {file = "attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b"}, +] + +[package.extras] +benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier"] +tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] + +[[package]] +name = "aws-cryptographic-material-providers" +version = "1.11.0" +description = "AWS Cryptographic Material Providers Library for Python" +optional = false +python-versions = "<4.0.0,>=3.11.0" +files = [ + {file = "aws_cryptographic_material_providers-1.11.0-py3-none-any.whl", hash = "sha256:9a9f0dca5b1902a4f16fb91cc1010dee74a721f84f411e81ffb4481fc0dd095f"}, + {file = "aws_cryptographic_material_providers-1.11.0.tar.gz", hash = "sha256:4ea5f9e5cc003e97d2ef98079dc25d8c49a0db01315ee887d19fd2f1c85ae9c3"}, +] + +[package.dependencies] +aws-cryptography-internal-dynamodb = "1.11.0" +aws-cryptography-internal-kms = "1.11.0" +aws-cryptography-internal-primitives = "1.11.0" +aws-cryptography-internal-standard-library = "1.11.0" + +[[package]] +name = "aws-cryptography-internal-dynamodb" +version = "1.11.0" +description = "" +optional = false +python-versions = "<4.0.0,>=3.11.0" +files = [ + {file = "aws_cryptography_internal_dynamodb-1.11.0-py3-none-any.whl", hash = "sha256:5a2da0ae6829d725f24018d001f4c733605f213820b723b6c75015843dc2427c"}, + {file = "aws_cryptography_internal_dynamodb-1.11.0.tar.gz", hash = "sha256:0800921ebb5dafc2853a2f5449f74aa03d24acd9ddb2ee58edca4002b97a5da5"}, +] + +[package.dependencies] +aws-cryptography-internal-standard-library = "1.11.0" +boto3 = ">=1.35.42,<2.0.0" + +[[package]] +name = "aws-cryptography-internal-kms" +version = "1.11.0" +description = "" +optional = false +python-versions = "<4.0.0,>=3.11.0" +files = [ + {file = "aws_cryptography_internal_kms-1.11.0-py3-none-any.whl", hash = "sha256:1c23cc8e970252fc7627868fc6b7a002400ec1d555ac29368e0eaddcceb07953"}, + {file = "aws_cryptography_internal_kms-1.11.0.tar.gz", hash = "sha256:a3ff5105b3e1c9d81e9698e0efc80de8a6bb8078b4512f9b39ed0f6161aae172"}, +] + +[package.dependencies] +aws-cryptography-internal-standard-library = "1.11.0" +boto3 = ">=1.35.42,<2.0.0" + +[[package]] +name = "aws-cryptography-internal-primitives" +version = "1.11.0" +description = "" +optional = false +python-versions = "<4.0.0,>=3.11.0" +files = [ + {file = "aws_cryptography_internal_primitives-1.11.0-py3-none-any.whl", hash = "sha256:84200885113f3534f4bff819ac1603c6d5c3bdd4d5c83a1b73ac2462cecec49b"}, + {file = "aws_cryptography_internal_primitives-1.11.0.tar.gz", hash = "sha256:9072af2c403b9e729dc767b44d1d642fa924a317a5bdbdffdf6dba0e93dc7996"}, +] + +[package.dependencies] +aws-cryptography-internal-standard-library = "1.11.0" +cryptography = ">=43.0.1,<46" + +[[package]] +name = "aws-cryptography-internal-standard-library" +version = "1.11.0" +description = "" +optional = false +python-versions = "<4.0.0,>=3.11.0" +files = [ + {file = "aws_cryptography_internal_standard_library-1.11.0-py3-none-any.whl", hash = "sha256:a2d5a4d8f70bce7242e8ebe06742223b8cd93253ed8081f44d7a8c1a086871e1"}, + {file = "aws_cryptography_internal_standard_library-1.11.0.tar.gz", hash = "sha256:36d82c6bc0361cf0ec3b7181804d375718f5c297949ddd902670f4452ecad3b0"}, +] + +[package.dependencies] +DafnyRuntimePython = "4.9.0" +pytz = ">=2023.3.post1,<2025.0.0" + +[[package]] +name = "boto3" +version = "1.39.4" +description = "The AWS SDK for Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "boto3-1.39.4-py3-none-any.whl", hash = "sha256:f8e9534b429121aa5c5b7c685c6a94dd33edf14f87926e9a182d5b50220ba284"}, + {file = "boto3-1.39.4.tar.gz", hash = "sha256:6c955729a1d70181bc8368e02a7d3f350884290def63815ebca8408ee6d47571"}, +] + +[package.dependencies] +botocore = ">=1.39.4,<1.40.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.13.0,<0.14.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.39.4" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.9" +files = [ + {file = "botocore-1.39.4-py3-none-any.whl", hash = "sha256:c41e167ce01cfd1973c3fa9856ef5244a51ddf9c82cb131120d8617913b6812a"}, + {file = "botocore-1.39.4.tar.gz", hash = "sha256:e662ac35c681f7942a93f2ec7b4cde8f8b56dd399da47a79fa3e370338521a56"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} + +[package.extras] +crt = ["awscrt (==0.23.8)"] + +[[package]] +name = "cffi" +version = "1.17.1" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, + {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be"}, + {file = "cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c"}, + {file = "cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15"}, + {file = "cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401"}, + {file = "cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"}, + {file = "cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655"}, + {file = "cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0"}, + {file = "cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4"}, + {file = "cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93"}, + {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3"}, + {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8"}, + {file = "cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65"}, + {file = "cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903"}, + {file = "cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e"}, + {file = "cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd"}, + {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed"}, + {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9"}, + {file = "cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d"}, + {file = "cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a"}, + {file = "cffi-1.17.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1"}, + {file = "cffi-1.17.1-cp38-cp38-win32.whl", hash = "sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8"}, + {file = "cffi-1.17.1-cp38-cp38-win_amd64.whl", hash = "sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1"}, + {file = "cffi-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16"}, + {file = "cffi-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e"}, + {file = "cffi-1.17.1-cp39-cp39-win32.whl", hash = "sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7"}, + {file = "cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662"}, + {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "click" +version = "8.2.1" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.10" +files = [ + {file = "click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b"}, + {file = "click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "coverage" +version = "7.8.2" +description = "Code coverage measurement for Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "coverage-7.8.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bd8ec21e1443fd7a447881332f7ce9d35b8fbd2849e761bb290b584535636b0a"}, + {file = "coverage-7.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4c26c2396674816deaeae7ded0e2b42c26537280f8fe313335858ffff35019be"}, + {file = "coverage-7.8.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1aec326ed237e5880bfe69ad41616d333712c7937bcefc1343145e972938f9b3"}, + {file = "coverage-7.8.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5e818796f71702d7a13e50c70de2a1924f729228580bcba1607cccf32eea46e6"}, + {file = "coverage-7.8.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:546e537d9e24efc765c9c891328f30f826e3e4808e31f5d0f87c4ba12bbd1622"}, + {file = "coverage-7.8.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ab9b09a2349f58e73f8ebc06fac546dd623e23b063e5398343c5270072e3201c"}, + {file = "coverage-7.8.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:fd51355ab8a372d89fb0e6a31719e825cf8df8b6724bee942fb5b92c3f016ba3"}, + {file = "coverage-7.8.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0774df1e093acb6c9e4d58bce7f86656aeed6c132a16e2337692c12786b32404"}, + {file = "coverage-7.8.2-cp310-cp310-win32.whl", hash = "sha256:00f2e2f2e37f47e5f54423aeefd6c32a7dbcedc033fcd3928a4f4948e8b96af7"}, + {file = "coverage-7.8.2-cp310-cp310-win_amd64.whl", hash = "sha256:145b07bea229821d51811bf15eeab346c236d523838eda395ea969d120d13347"}, + {file = "coverage-7.8.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b99058eef42e6a8dcd135afb068b3d53aff3921ce699e127602efff9956457a9"}, + {file = "coverage-7.8.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5feb7f2c3e6ea94d3b877def0270dff0947b8d8c04cfa34a17be0a4dc1836879"}, + {file = "coverage-7.8.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:670a13249b957bb9050fab12d86acef7bf8f6a879b9d1a883799276e0d4c674a"}, + {file = "coverage-7.8.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0bdc8bf760459a4a4187b452213e04d039990211f98644c7292adf1e471162b5"}, + {file = "coverage-7.8.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07a989c867986c2a75f158f03fdb413128aad29aca9d4dbce5fc755672d96f11"}, + {file = "coverage-7.8.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2db10dedeb619a771ef0e2949ccba7b75e33905de959c2643a4607bef2f3fb3a"}, + {file = "coverage-7.8.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e6ea7dba4e92926b7b5f0990634b78ea02f208d04af520c73a7c876d5a8d36cb"}, + {file = "coverage-7.8.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ef2f22795a7aca99fc3c84393a55a53dd18ab8c93fb431004e4d8f0774150f54"}, + {file = "coverage-7.8.2-cp311-cp311-win32.whl", hash = "sha256:641988828bc18a6368fe72355df5f1703e44411adbe49bba5644b941ce6f2e3a"}, + {file = "coverage-7.8.2-cp311-cp311-win_amd64.whl", hash = "sha256:8ab4a51cb39dc1933ba627e0875046d150e88478dbe22ce145a68393e9652975"}, + {file = "coverage-7.8.2-cp311-cp311-win_arm64.whl", hash = "sha256:8966a821e2083c74d88cca5b7dcccc0a3a888a596a04c0b9668a891de3a0cc53"}, + {file = "coverage-7.8.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e2f6fe3654468d061942591aef56686131335b7a8325684eda85dacdf311356c"}, + {file = "coverage-7.8.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:76090fab50610798cc05241bf83b603477c40ee87acd358b66196ab0ca44ffa1"}, + {file = "coverage-7.8.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bd0a0a5054be160777a7920b731a0570284db5142abaaf81bcbb282b8d99279"}, + {file = "coverage-7.8.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da23ce9a3d356d0affe9c7036030b5c8f14556bd970c9b224f9c8205505e3b99"}, + {file = "coverage-7.8.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9392773cffeb8d7e042a7b15b82a414011e9d2b5fdbbd3f7e6a6b17d5e21b20"}, + {file = "coverage-7.8.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:876cbfd0b09ce09d81585d266c07a32657beb3eaec896f39484b631555be0fe2"}, + {file = "coverage-7.8.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3da9b771c98977a13fbc3830f6caa85cae6c9c83911d24cb2d218e9394259c57"}, + {file = "coverage-7.8.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9a990f6510b3292686713bfef26d0049cd63b9c7bb17e0864f133cbfd2e6167f"}, + {file = "coverage-7.8.2-cp312-cp312-win32.whl", hash = "sha256:bf8111cddd0f2b54d34e96613e7fbdd59a673f0cf5574b61134ae75b6f5a33b8"}, + {file = "coverage-7.8.2-cp312-cp312-win_amd64.whl", hash = "sha256:86a323a275e9e44cdf228af9b71c5030861d4d2610886ab920d9945672a81223"}, + {file = "coverage-7.8.2-cp312-cp312-win_arm64.whl", hash = "sha256:820157de3a589e992689ffcda8639fbabb313b323d26388d02e154164c57b07f"}, + {file = "coverage-7.8.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ea561010914ec1c26ab4188aef8b1567272ef6de096312716f90e5baa79ef8ca"}, + {file = "coverage-7.8.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cb86337a4fcdd0e598ff2caeb513ac604d2f3da6d53df2c8e368e07ee38e277d"}, + {file = "coverage-7.8.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26a4636ddb666971345541b59899e969f3b301143dd86b0ddbb570bd591f1e85"}, + {file = "coverage-7.8.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5040536cf9b13fb033f76bcb5e1e5cb3b57c4807fef37db9e0ed129c6a094257"}, + {file = "coverage-7.8.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc67994df9bcd7e0150a47ef41278b9e0a0ea187caba72414b71dc590b99a108"}, + {file = "coverage-7.8.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6e6c86888fd076d9e0fe848af0a2142bf606044dc5ceee0aa9eddb56e26895a0"}, + {file = "coverage-7.8.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:684ca9f58119b8e26bef860db33524ae0365601492e86ba0b71d513f525e7050"}, + {file = "coverage-7.8.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8165584ddedb49204c4e18da083913bdf6a982bfb558632a79bdaadcdafd0d48"}, + {file = "coverage-7.8.2-cp313-cp313-win32.whl", hash = "sha256:34759ee2c65362163699cc917bdb2a54114dd06d19bab860725f94ef45a3d9b7"}, + {file = "coverage-7.8.2-cp313-cp313-win_amd64.whl", hash = "sha256:2f9bc608fbafaee40eb60a9a53dbfb90f53cc66d3d32c2849dc27cf5638a21e3"}, + {file = "coverage-7.8.2-cp313-cp313-win_arm64.whl", hash = "sha256:9fe449ee461a3b0c7105690419d0b0aba1232f4ff6d120a9e241e58a556733f7"}, + {file = "coverage-7.8.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:8369a7c8ef66bded2b6484053749ff220dbf83cba84f3398c84c51a6f748a008"}, + {file = "coverage-7.8.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:159b81df53a5fcbc7d45dae3adad554fdbde9829a994e15227b3f9d816d00b36"}, + {file = "coverage-7.8.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e6fcbbd35a96192d042c691c9e0c49ef54bd7ed865846a3c9d624c30bb67ce46"}, + {file = "coverage-7.8.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05364b9cc82f138cc86128dc4e2e1251c2981a2218bfcd556fe6b0fbaa3501be"}, + {file = "coverage-7.8.2-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46d532db4e5ff3979ce47d18e2fe8ecad283eeb7367726da0e5ef88e4fe64740"}, + {file = "coverage-7.8.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4000a31c34932e7e4fa0381a3d6deb43dc0c8f458e3e7ea6502e6238e10be625"}, + {file = "coverage-7.8.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:43ff5033d657cd51f83015c3b7a443287250dc14e69910577c3e03bd2e06f27b"}, + {file = "coverage-7.8.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:94316e13f0981cbbba132c1f9f365cac1d26716aaac130866ca812006f662199"}, + {file = "coverage-7.8.2-cp313-cp313t-win32.whl", hash = "sha256:3f5673888d3676d0a745c3d0e16da338c5eea300cb1f4ada9c872981265e76d8"}, + {file = "coverage-7.8.2-cp313-cp313t-win_amd64.whl", hash = "sha256:2c08b05ee8d7861e45dc5a2cc4195c8c66dca5ac613144eb6ebeaff2d502e73d"}, + {file = "coverage-7.8.2-cp313-cp313t-win_arm64.whl", hash = "sha256:1e1448bb72b387755e1ff3ef1268a06617afd94188164960dba8d0245a46004b"}, + {file = "coverage-7.8.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:496948261eaac5ac9cf43f5d0a9f6eb7a6d4cb3bedb2c5d294138142f5c18f2a"}, + {file = "coverage-7.8.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eacd2de0d30871eff893bab0b67840a96445edcb3c8fd915e6b11ac4b2f3fa6d"}, + {file = "coverage-7.8.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b039ffddc99ad65d5078ef300e0c7eed08c270dc26570440e3ef18beb816c1ca"}, + {file = "coverage-7.8.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e49824808d4375ede9dd84e9961a59c47f9113039f1a525e6be170aa4f5c34d"}, + {file = "coverage-7.8.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b069938961dfad881dc2f8d02b47645cd2f455d3809ba92a8a687bf513839787"}, + {file = "coverage-7.8.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:de77c3ba8bb686d1c411e78ee1b97e6e0b963fb98b1637658dd9ad2c875cf9d7"}, + {file = "coverage-7.8.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:1676628065a498943bd3f64f099bb573e08cf1bc6088bbe33cf4424e0876f4b3"}, + {file = "coverage-7.8.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:8e1a26e7e50076e35f7afafde570ca2b4d7900a491174ca357d29dece5aacee7"}, + {file = "coverage-7.8.2-cp39-cp39-win32.whl", hash = "sha256:6782a12bf76fa61ad9350d5a6ef5f3f020b57f5e6305cbc663803f2ebd0f270a"}, + {file = "coverage-7.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:1efa4166ba75ccefd647f2d78b64f53f14fb82622bc94c5a5cb0a622f50f1c9e"}, + {file = "coverage-7.8.2-pp39.pp310.pp311-none-any.whl", hash = "sha256:ec455eedf3ba0bbdf8f5a570012617eb305c63cb9f03428d39bf544cb2b94837"}, + {file = "coverage-7.8.2-py3-none-any.whl", hash = "sha256:726f32ee3713f7359696331a18daf0c3b3a70bb0ae71141b9d3c52be7c595e32"}, + {file = "coverage-7.8.2.tar.gz", hash = "sha256:a886d531373a1f6ff9fad2a2ba4a045b68467b779ae729ee0b3b10ac20033b27"}, +] + +[package.extras] +toml = ["tomli"] + +[[package]] +name = "cryptography" +version = "43.0.3" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-43.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e1ce50266f4f70bf41a2c6dc4358afadae90e2a1e5342d3c08883df1675374f"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:443c4a81bb10daed9a8f334365fe52542771f25aedaf889fd323a853ce7377d6"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:74f57f24754fe349223792466a709f8e0c093205ff0dca557af51072ff47ab18"}, + {file = "cryptography-43.0.3-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9762ea51a8fc2a88b70cf2995e5675b38d93bf36bd67d91721c309df184f49bd"}, + {file = "cryptography-43.0.3-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:81ef806b1fef6b06dcebad789f988d3b37ccaee225695cf3e07648eee0fc6b73"}, + {file = "cryptography-43.0.3-cp37-abi3-win32.whl", hash = "sha256:cbeb489927bd7af4aa98d4b261af9a5bc025bd87f0e3547e11584be9e9427be2"}, + {file = "cryptography-43.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:f46304d6f0c6ab8e52770addfa2fc41e6629495548862279641972b6215451cd"}, + {file = "cryptography-43.0.3-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:8ac43ae87929a5982f5948ceda07001ee5e83227fd69cf55b109144938d96984"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:846da004a5804145a5f441b8530b4bf35afbf7da70f82409f151695b127213d5"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f996e7268af62598f2fc1204afa98a3b5712313a55c4c9d434aef49cadc91d4"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f7b178f11ed3664fd0e995a47ed2b5ff0a12d893e41dd0494f406d1cf555cab7"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:c2e6fc39c4ab499049df3bdf567f768a723a5e8464816e8f009f121a5a9f4405"}, + {file = "cryptography-43.0.3-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e1be4655c7ef6e1bbe6b5d0403526601323420bcf414598955968c9ef3eb7d16"}, + {file = "cryptography-43.0.3-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:df6b6c6d742395dd77a23ea3728ab62f98379eff8fb61be2744d4679ab678f73"}, + {file = "cryptography-43.0.3-cp39-abi3-win32.whl", hash = "sha256:d56e96520b1020449bbace2b78b603442e7e378a9b3bd68de65c782db1507995"}, + {file = "cryptography-43.0.3-cp39-abi3-win_amd64.whl", hash = "sha256:0c580952eef9bf68c4747774cde7ec1d85a6e61de97281f2dba83c7d2c806362"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:d03b5621a135bffecad2c73e9f4deb1a0f977b9a8ffe6f8e002bf6c9d07b918c"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a2a431ee15799d6db9fe80c82b055bae5a752bef645bba795e8e52687c69efe3"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:281c945d0e28c92ca5e5930664c1cefd85efe80e5c0d2bc58dd63383fda29f83"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f18c716be16bc1fea8e95def49edf46b82fccaa88587a45f8dc0ff6ab5d8e0a7"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:4a02ded6cd4f0a5562a8887df8b3bd14e822a90f97ac5e544c162899bc467664"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:53a583b6637ab4c4e3591a15bc9db855b8d9dee9a669b550f311480acab6eb08"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1ec0bcf7e17c0c5669d881b1cd38c4972fade441b27bda1051665faaa89bdcaa"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2ce6fae5bdad59577b44e4dfed356944fbf1d925269114c28be377692643b4ff"}, + {file = "cryptography-43.0.3.tar.gz", hash = "sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805"}, +] + +[package.dependencies] +cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] +nox = ["nox"] +pep8test = ["check-sdist", "click", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["certifi", "cryptography-vectors (==43.0.3)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + +[[package]] +name = "dafnyruntimepython" +version = "4.9.0" +description = "Dafny runtime for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "DafnyRuntimePython-4.9.0-py3-none-any.whl", hash = "sha256:c9cdcf127f5b6a4c6c9cf69016b9486318c3a6600e7f03fcbc621f6a5398479c"}, + {file = "dafnyruntimepython-4.9.0.tar.gz", hash = "sha256:03a4c2dbbe45c13dc2c7dbefad01812367b3bb217a14b4b848d7e94ef5c08cee"}, +] + +[[package]] +name = "fastapi" +version = "0.115.12" +description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fastapi-0.115.12-py3-none-any.whl", hash = "sha256:e94613d6c05e27be7ffebdd6ea5f388112e5e430c8f7d6494a9d1d88d43e814d"}, + {file = "fastapi-0.115.12.tar.gz", hash = "sha256:1e2c2a2646905f9e83d32f04a3f86aff4a286669c6c950ca95b5fd68c2602681"}, +] + +[package.dependencies] +pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<2.1.0 || >2.1.0,<3.0.0" +starlette = ">=0.40.0,<0.47.0" +typing-extensions = ">=4.8.0" + +[package.extras] +all = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.5)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=3.1.5)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] +standard = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.5)", "httpx (>=0.23.0)", "jinja2 (>=3.1.5)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"] + +[[package]] +name = "h11" +version = "0.16.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.8" +files = [ + {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, + {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, +] + +[[package]] +name = "idna" +version = "3.10" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.6" +files = [ + {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, + {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, +] + +[package.extras] +all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] + +[[package]] +name = "iniconfig" +version = "2.1.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.8" +files = [ + {file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"}, + {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, +] + +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + +[[package]] +name = "packaging" +version = "25.0" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, + {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, + {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["coverage", "pytest", "pytest-benchmark"] + +[[package]] +name = "pycparser" +version = "2.22" +description = "C parser in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, + {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, +] + +[[package]] +name = "pydantic" +version = "2.11.5" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pydantic-2.11.5-py3-none-any.whl", hash = "sha256:f9c26ba06f9747749ca1e5c94d6a85cb84254577553c8785576fd38fa64dc0f7"}, + {file = "pydantic-2.11.5.tar.gz", hash = "sha256:7f853db3d0ce78ce8bbb148c401c2cdd6431b3473c0cdff2755c7690952a7b7a"}, +] + +[package.dependencies] +annotated-types = ">=0.6.0" +pydantic-core = "2.33.2" +typing-extensions = ">=4.12.2" +typing-inspection = ">=0.4.0" + +[package.extras] +email = ["email-validator (>=2.0.0)"] +timezone = ["tzdata"] + +[[package]] +name = "pydantic-core" +version = "2.33.2" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pydantic_core-2.33.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8"}, + {file = "pydantic_core-2.33.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d"}, + {file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d"}, + {file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572"}, + {file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02"}, + {file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b"}, + {file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2"}, + {file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a"}, + {file = "pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac"}, + {file = "pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a"}, + {file = "pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b"}, + {file = "pydantic_core-2.33.2-cp310-cp310-win32.whl", hash = "sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22"}, + {file = "pydantic_core-2.33.2-cp310-cp310-win_amd64.whl", hash = "sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640"}, + {file = "pydantic_core-2.33.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7"}, + {file = "pydantic_core-2.33.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246"}, + {file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f"}, + {file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc"}, + {file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de"}, + {file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a"}, + {file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef"}, + {file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e"}, + {file = "pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d"}, + {file = "pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30"}, + {file = "pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf"}, + {file = "pydantic_core-2.33.2-cp311-cp311-win32.whl", hash = "sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51"}, + {file = "pydantic_core-2.33.2-cp311-cp311-win_amd64.whl", hash = "sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab"}, + {file = "pydantic_core-2.33.2-cp311-cp311-win_arm64.whl", hash = "sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65"}, + {file = "pydantic_core-2.33.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc"}, + {file = "pydantic_core-2.33.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7"}, + {file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025"}, + {file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011"}, + {file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f"}, + {file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88"}, + {file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1"}, + {file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b"}, + {file = "pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1"}, + {file = "pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6"}, + {file = "pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea"}, + {file = "pydantic_core-2.33.2-cp312-cp312-win32.whl", hash = "sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290"}, + {file = "pydantic_core-2.33.2-cp312-cp312-win_amd64.whl", hash = "sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2"}, + {file = "pydantic_core-2.33.2-cp312-cp312-win_arm64.whl", hash = "sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab"}, + {file = "pydantic_core-2.33.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f"}, + {file = "pydantic_core-2.33.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6"}, + {file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef"}, + {file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a"}, + {file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916"}, + {file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a"}, + {file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d"}, + {file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56"}, + {file = "pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5"}, + {file = "pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e"}, + {file = "pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162"}, + {file = "pydantic_core-2.33.2-cp313-cp313-win32.whl", hash = "sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849"}, + {file = "pydantic_core-2.33.2-cp313-cp313-win_amd64.whl", hash = "sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9"}, + {file = "pydantic_core-2.33.2-cp313-cp313-win_arm64.whl", hash = "sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9"}, + {file = "pydantic_core-2.33.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac"}, + {file = "pydantic_core-2.33.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5"}, + {file = "pydantic_core-2.33.2-cp313-cp313t-win_amd64.whl", hash = "sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9"}, + {file = "pydantic_core-2.33.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d"}, + {file = "pydantic_core-2.33.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954"}, + {file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb"}, + {file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7"}, + {file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4"}, + {file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b"}, + {file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3"}, + {file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a"}, + {file = "pydantic_core-2.33.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782"}, + {file = "pydantic_core-2.33.2-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9"}, + {file = "pydantic_core-2.33.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e"}, + {file = "pydantic_core-2.33.2-cp39-cp39-win32.whl", hash = "sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9"}, + {file = "pydantic_core-2.33.2-cp39-cp39-win_amd64.whl", hash = "sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27"}, + {file = "pydantic_core-2.33.2.tar.gz", hash = "sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + +[[package]] +name = "pygments" +version = "2.19.2" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, + {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, +] + +[package.extras] +windows-terminal = ["colorama (>=0.4.6)"] + +[[package]] +name = "pytest" +version = "8.4.1" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7"}, + {file = "pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c"}, +] + +[package.dependencies] +colorama = {version = ">=0.4", markers = "sys_platform == \"win32\""} +iniconfig = ">=1" +packaging = ">=20" +pluggy = ">=1.5,<2" +pygments = ">=2.7.2" + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-cov" +version = "6.1.1" +description = "Pytest plugin for measuring coverage." +optional = false +python-versions = ">=3.9" +files = [ + {file = "pytest_cov-6.1.1-py3-none-any.whl", hash = "sha256:bddf29ed2d0ab6f4df17b4c55b0a657287db8684af9c42ea546b21b1041b3dde"}, + {file = "pytest_cov-6.1.1.tar.gz", hash = "sha256:46935f7aaefba760e716c2ebfbe1c216240b9592966e7da99ea8292d4d3e2a0a"}, +] + +[package.dependencies] +coverage = {version = ">=7.5", extras = ["toml"]} +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2024.2" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, + {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, +] + +[[package]] +name = "s3transfer" +version = "0.13.0" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">=3.9" +files = [ + {file = "s3transfer-0.13.0-py3-none-any.whl", hash = "sha256:0148ef34d6dd964d0d8cf4311b2b21c474693e57c2e069ec708ce043d2b527be"}, + {file = "s3transfer-0.13.0.tar.gz", hash = "sha256:f5e6db74eb7776a37208001113ea7aa97695368242b364d73e91c981ac522177"}, +] + +[package.dependencies] +botocore = ">=1.37.4,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] + +[[package]] +name = "six" +version = "1.17.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, + {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, +] + +[[package]] +name = "starlette" +version = "0.46.2" +description = "The little ASGI library that shines." +optional = false +python-versions = ">=3.9" +files = [ + {file = "starlette-0.46.2-py3-none-any.whl", hash = "sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35"}, + {file = "starlette-0.46.2.tar.gz", hash = "sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5"}, +] + +[package.dependencies] +anyio = ">=3.6.2,<5" + +[package.extras] +full = ["httpx (>=0.27.0,<0.29.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.18)", "pyyaml"] + +[[package]] +name = "typing-extensions" +version = "4.13.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c"}, + {file = "typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef"}, +] + +[[package]] +name = "typing-inspection" +version = "0.4.1" +description = "Runtime typing introspection tools" +optional = false +python-versions = ">=3.9" +files = [ + {file = "typing_inspection-0.4.1-py3-none-any.whl", hash = "sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51"}, + {file = "typing_inspection-0.4.1.tar.gz", hash = "sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28"}, +] + +[package.dependencies] +typing-extensions = ">=4.12.0" + +[[package]] +name = "urllib3" +version = "2.5.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.9" +files = [ + {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, + {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "uvicorn" +version = "0.34.2" +description = "The lightning-fast ASGI server." +optional = false +python-versions = ">=3.9" +files = [ + {file = "uvicorn-0.34.2-py3-none-any.whl", hash = "sha256:deb49af569084536d269fe0a6d67e3754f104cf03aba7c11c40f01aadf33c403"}, + {file = "uvicorn-0.34.2.tar.gz", hash = "sha256:0e929828f6186353a80b58ea719861d2629d766293b6d19baf086ba31d4f3328"}, +] + +[package.dependencies] +click = ">=7.0" +h11 = ">=0.8" + +[package.extras] +standard = ["colorama (>=0.4)", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.11" +content-hash = "cb96fac2ddbdb9fc156d6f957ff76f565da35fcee31f1a4ed085676e0e175509" diff --git a/test-server/python-server/pyproject.toml b/test-server/python-server/pyproject.toml index a4d2e4ab..7eb8742d 100644 --- a/test-server/python-server/pyproject.toml +++ b/test-server/python-server/pyproject.toml @@ -1,22 +1,24 @@ -[tool.poetry] +[project] name = "python-server" version = "0.1.0" description = "A Python server implementation" -authors = ["Your Name"] +authors = [ + {name = "Your Name"} +] readme = "README.md" -package-mode = false +requires-python = ">=3.11" +dependencies = [ + "boto3>=1.37.2", + "pytest>=8.4.1,<9.0.0", + "fastapi>=0.115.12", + "uvicorn>=0.34.2", +] -[tool.poetry.dependencies] -python = "^3.11" -boto3 = "^1.37.2" -pytest = ">=8.4.1,<9.0.0" -fastapi = "^0.115.12" -uvicorn = "^0.34.2" -amazon-s3-encryption-client-python = { path = "../..", develop = true } - -[tool.poetry.group.dev.dependencies] -pytest-cov = "^6.1.1" +[project.optional-dependencies] +dev = [ + "pytest-cov>=6.1.1", +] [build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" From a26d99bb6003516b2e8eb520bd3879fb7cee061e Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 3 Sep 2025 14:13:12 -0700 Subject: [PATCH 50/81] address feedback --- test-server/OPTIMIZATION.md | 76 ------------------- test-server/README.md | 2 +- .../encryption/s3/S3ECJavaTestServer.java | 1 - .../amazon/encryption/s3/RoundTripTests.java | 19 +++-- test-server/python-server/README.md | 8 +- test-server/python-server/pyproject.toml | 4 +- test-server/python-server/src/main.py | 9 +-- 7 files changed, 14 insertions(+), 105 deletions(-) delete mode 100644 test-server/OPTIMIZATION.md diff --git a/test-server/OPTIMIZATION.md b/test-server/OPTIMIZATION.md deleted file mode 100644 index 7774cf78..00000000 --- a/test-server/OPTIMIZATION.md +++ /dev/null @@ -1,76 +0,0 @@ -# Test Server Performance Optimizations - -This document describes the performance optimizations implemented to speed up the test-server CI process. - -## Overview - -The test-server CI process involves starting both Python and Java servers, then running Java tests against them. The original implementation was taking over 5 minutes to run, with most of the time spent on Gradle/Java setup rather than the actual tests. - -## Optimizations Implemented - -### 1. Parallel Server Startup - -- Updated the `start-servers` target in the Makefile to start both Python and Java servers concurrently -- Updated the `ci` target to use parallel server startup - -### 2. Gradle Performance Optimizations - -- Added Gradle build caching -- Enabled parallel execution of Gradle tasks -- Configured the Gradle daemon for faster startup -- Optimized JVM memory settings -- Added incremental compilation -- Configured parallel test execution - -### 3. CI Workflow Optimizations - -- Added caching for Gradle dependencies and build outputs -- Added caching for uv dependencies -- Set environment variables to ensure Gradle optimizations are used - -## Configuration Files - -The following files were modified or created: - -1. `test-server/Makefile`: Added new targets for parallel execution -2. `.github/workflows/test.yml`: Added caching and updated to use the optimized CI target -3. `test-server/java-server/gradle.properties` and `test-server/java-tests/gradle.properties`: Added performance settings -4. `test-server/gradle.init`: Added global Gradle settings for all projects - -## Usage - -### Local Development - -For local development and testing, you can use the optimized targets: - -```bash -# Run the CI process (now optimized by default) -cd test-server && make ci - -# Start servers in parallel -cd test-server && make start-servers - -# Run tests with optimized Gradle settings -cd test-server/java-tests && ./gradlew --build-cache --parallel integ -``` - -### CI Environment - -The GitHub Actions workflow has been updated to use the optimized CI process automatically. - -## Performance Impact - -The optimizations are expected to significantly reduce the CI execution time by: - -1. Running server startup in parallel (saves time equal to the slower of the two servers) -2. Caching Gradle and uv dependencies (saves download and resolution time) -3. Optimizing Gradle execution (reduces build time) -4. Enabling incremental compilation (reduces compilation time on subsequent runs) - -## Troubleshooting - -If you encounter issues with the CI process: - -1. Check Gradle daemon logs: `cat ~/.gradle/daemon/*/daemon-*.out.log` -2. Disable specific optimizations by modifying the relevant configuration files -3. Try running the servers sequentially by modifying the Makefile diff --git a/test-server/README.md b/test-server/README.md index ca8ee6a7..a320d1d1 100644 --- a/test-server/README.md +++ b/test-server/README.md @@ -4,7 +4,7 @@ Or G-RTFM. Or something. ## What? -This is an attempt at writing a write-once, run-multiple test server. +This is a write-once, run-multiple test server. ## How? diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java b/test-server/java-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java index 327966c6..8ad437f4 100644 --- a/test-server/java-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java +++ b/test-server/java-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java @@ -6,7 +6,6 @@ package software.amazon.encryption.s3; import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.encryption.s3.S3EncryptionClient; import java.net.URI; import java.util.Map; diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index ff50fc85..211269d7 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -73,8 +73,8 @@ public class RoundTripTests { } static public class LanguageServerTarget { - public String getLangaugeName() { - return langaugeName; + public String getLanguageName() { + return languageName; } public URI getServerURI() { @@ -82,7 +82,7 @@ public URI getServerURI() { } private final String baseURI = "http://localhost"; - private String langaugeName; + private String languageName; private URI serverURI; @Override @@ -92,22 +92,22 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; LanguageServerTarget that = (LanguageServerTarget) o; - return Objects.equals(langaugeName, that.langaugeName) && Objects.equals(serverURI, that.serverURI); + return Objects.equals(languageName, that.languageName) && Objects.equals(serverURI, that.serverURI); } @Override public int hashCode() { - return Objects.hash(langaugeName, serverURI); + return Objects.hash(languageName, serverURI); } LanguageServerTarget(String language, String port) { - langaugeName = language; + languageName = language; serverURI = URI.create(baseURI+ ":" + port); } @Override public String toString() { - return langaugeName; + return languageName; } } @@ -116,7 +116,7 @@ public static void setup() { // Wait for servers to start for (LanguageServerTarget server : serverList) { if (!serverListening(server.getServerURI())) { - throw new RuntimeException(String.format("Test Server for %s is not running at endpoint: %s", server.getLangaugeName(), server.getServerURI())); + throw new RuntimeException(String.format("Test Server for %s is not running at endpoint: %s", server.getLanguageName(), server.getServerURI())); } } } @@ -145,14 +145,13 @@ static S3ECTestServerClient testServerClientFor(LanguageServerTarget server) { static Stream clientsForTest() { return serverList.stream() - .map(LanguageServerTarget::getLangaugeName) + .map(LanguageServerTarget::getLanguageName) .map(Arguments::of); } static Stream crossLanguageClients() { return serverList.stream() .flatMap(t1 -> serverList.stream() -// .filter(t2 -> !t1.equals(t2)) .flatMap(t2 -> Stream.of( Arguments.of(t1, t2) ))); diff --git a/test-server/python-server/README.md b/test-server/python-server/README.md index f9e4fd5b..93f8468b 100644 --- a/test-server/python-server/README.md +++ b/test-server/python-server/README.md @@ -31,13 +31,7 @@ uv pip install -e ../.. .venv/bin/python src/main.py ``` -The server will start on `http://localhost:8080` with the following endpoints: -- `GET /` - Welcome message -- `POST /get-beer` - Get a beer with specified ID - - Request body: `{"Id": "string"}` - - Response: `{"beer": "beer{Id}"}` -- `GET /docs` - Interactive API documentation (provided by Swagger UI) -- `GET /redoc` - Alternative API documentation (provided by ReDoc) +The server will start on `http://localhost:8081`. ## Running Tests diff --git a/test-server/python-server/pyproject.toml b/test-server/python-server/pyproject.toml index 7eb8742d..2ae329e4 100644 --- a/test-server/python-server/pyproject.toml +++ b/test-server/python-server/pyproject.toml @@ -1,9 +1,9 @@ [project] name = "python-server" version = "0.1.0" -description = "A Python server implementation" +description = "Python implementation of S3ECTestServer" authors = [ - {name = "Your Name"} + {name = "AWS Crypto Tools"} ] readme = "README.md" requires-python = ">=3.11" diff --git a/test-server/python-server/src/main.py b/test-server/python-server/src/main.py index 13ddbf5e..6c57c6bd 100755 --- a/test-server/python-server/src/main.py +++ b/test-server/python-server/src/main.py @@ -154,7 +154,7 @@ async def get_object(bucket: str, key: str, request: Request): except S3EncryptionClientError as ex: return create_s3_encryption_client_error(str(ex)) except Exception as e: - return create_generic_server_error(e) + return create_generic_server_error(str(e)) @app.post("/client") @@ -186,13 +186,6 @@ async def client_endpoint(request: Request): ) wrapped_client = boto3.client("s3") client_config = S3EncryptionClientConfig(keyring) - # Create S3EncryptionClientConfig - # client_config = S3EncryptionClientConfig( - # enable_legacy_unauthenticated_modes=config_data.get("enableLegacyUnauthenticatedModes", False), - # enable_delayed_authentication_mode=config_data.get("enableDelayedAuthenticationMode", False), - # enable_legacy_wrapping_algorithms=config_data.get("enableLegacyWrappingAlgorithms", False), - # buffer_size=config_data.get("setBufferSize", 0) - # ) # Create S3EncryptionClient client = S3EncryptionClient(wrapped_client, client_config) From 2a79383a089fb321e641e0b473bc53dc32b7bf66 Mon Sep 17 00:00:00 2001 From: Tony Knapp <5892063+texastony@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:21:51 -0600 Subject: [PATCH 51/81] chore[dev]: replace poetry with UV in README (#121) * chore[dev]: replace poetry with UV in README * fix[ci]: run-tests in main.yml needs permissions * chore[dev]: Add PR helper to instruction (#122) * chore: resolve PR feedback * chore: suppress docstring and line length linting rules for test files --- .github/workflows/main.yml | 5 ++++- README.md | 12 +++++++++++- pyproject.toml | 6 +++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e10b7d0d..675a0099 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,7 +2,7 @@ name: Main Workflow on: push: - branches: [ main ] + branches: [ staging ] pull_request: workflow_dispatch: inputs: @@ -22,4 +22,7 @@ jobs: uses: ./.github/workflows/test.yml with: python-version: ${{ inputs.python-version || '3.11' }} + permissions: + id-token: write + contents: read secrets: inherit diff --git a/README.md b/README.md index c24d1796..edcfd4ff 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This library provides an S3 client that supports client-side encryption. ### Prerequisites - Python 3.11 or higher -- [Poetry](https://python-poetry.org/) for dependency management +- [uv](https://github.com/astral-sh/uv) for package and project management ### Setup @@ -73,3 +73,13 @@ Common Flake8 issues in the codebase include: - **Code complexity** (C901): Refactor complex functions When contributing to this project, please try to fix linting issues in the files you modify. + +### Pull Request Command +While this project is in development, +it is useful to use `gh pr` to create the pull-requests, +so they can be associated with the GitHub project. + +```sh +gh pr create -B staging -p "S3EC-Python" -f +``` + diff --git a/pyproject.toml b/pyproject.toml index 883b6914..187b99a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,8 @@ test = [ "pytest>=8.4.1", ] dev = [ - "black>=24.3.0", + # black >= 26 requires a reformat + "black>=24.3.0,<26.0.0", "ruff>=0.3.0", ] @@ -52,3 +53,6 @@ max-complexity = 10 [tool.ruff.lint.isort] known-first-party = ["s3_encryption"] + +[tool.ruff.lint.per-file-ignores] +"test/**/*.py" = ["D100", "D101", "D102", "D103", "D104", "E501"] From 894aeb9b63ad0cdf23ceafcc90315c82cac18d58 Mon Sep 17 00:00:00 2001 From: Tony Knapp <5892063+texastony@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:41:31 -0600 Subject: [PATCH 52/81] fix(exceptions): properly initialize BotoCoreError subclasses with message formatting (#123) * feat: Exceptions * some duvet stuff we may not need * fix(exceptions): properly initialize BotoCoreError subclasses with message formatting Add BotoCoreError import and implement proper exception initialization: - Add fmt class attribute with {msg} placeholder for message formatting - Override __init__ to accept message parameter with sensible defaults - Add docstrings to __init__ methods - Add unit tests for default, custom exception messages, and inherentance. kiro-clie Cost: ~110,565 tokens Fixes syntax errors and TypeError when raising exceptions with messages. * chore: remove incomplete duvet work * Trigger CI build --- src/s3_encryption/exceptions.py | 22 ++++++++++++-- test/test_exceptions.py | 51 +++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 test/test_exceptions.py diff --git a/src/s3_encryption/exceptions.py b/src/s3_encryption/exceptions.py index 748075fc..463a180d 100644 --- a/src/s3_encryption/exceptions.py +++ b/src/s3_encryption/exceptions.py @@ -5,6 +5,24 @@ This module contains custom exception classes used throughout the S3 Encryption Client. """ +from botocore.exceptions import BotoCoreError -class S3EncryptionClientError(Exception): - """Exception class for S3 Encryption Client errors.""" + +class S3EncryptionClientError(BotoCoreError): + """Exception class for non-Security S3 Encryption Client errors.""" + + fmt = "{msg}" + + def __init__(self, message="An unspecified S3 Encryption Client error occurred"): + """Initialize the exception with a message.""" + super().__init__(msg=message) + + +class S3EncryptionClientSecurityError(BotoCoreError): + """Security Exceptions for S3 Encryption Client errors.""" + + fmt = "{msg}" + + def __init__(self, message="An unspecified S3 Encryption Client Security error occurred"): + """Initialize the exception with a message.""" + super().__init__(msg=message) diff --git a/test/test_exceptions.py b/test/test_exceptions.py new file mode 100644 index 00000000..f93e3d9d --- /dev/null +++ b/test/test_exceptions.py @@ -0,0 +1,51 @@ +import pytest +from botocore.exceptions import BotoCoreError + +from s3_encryption.exceptions import ( + S3EncryptionClientError, + S3EncryptionClientSecurityError, +) + + +class TestS3EncryptionClientError: + def test_default_message(self): + error = S3EncryptionClientError() + assert str(error) == "An unspecified S3 Encryption Client error occurred" + + def test_custom_message(self): + error = S3EncryptionClientError("Custom error message") + assert str(error) == "Custom error message" + + def test_empty_message(self): + error = S3EncryptionClientError("") + assert str(error) == "" + + def test_inherits_from_botocore_error(self): + error = S3EncryptionClientError("test") + assert isinstance(error, BotoCoreError) + + def test_can_be_caught_as_botocore_error(self): + with pytest.raises(BotoCoreError): + raise S3EncryptionClientError("test error") + + +class TestS3EncryptionClientSecurityError: + def test_default_message(self): + error = S3EncryptionClientSecurityError() + assert str(error) == "An unspecified S3 Encryption Client Security error occurred" + + def test_custom_message(self): + error = S3EncryptionClientSecurityError("Custom security error") + assert str(error) == "Custom security error" + + def test_empty_message(self): + error = S3EncryptionClientSecurityError("") + assert str(error) == "" + + def test_inherits_from_botocore_error(self): + error = S3EncryptionClientSecurityError("test") + assert isinstance(error, BotoCoreError) + + def test_can_be_caught_as_botocore_error(self): + with pytest.raises(BotoCoreError): + raise S3EncryptionClientSecurityError("test security error") From 0fde32f9af47e54f61aa9dff0d7e2cbd86f00638 Mon Sep 17 00:00:00 2001 From: Tony Knapp <5892063+texastony@users.noreply.github.com> Date: Wed, 28 Jan 2026 14:58:39 -0600 Subject: [PATCH 53/81] chore(deps): bump black, reformat for black bump * chore(deps): bump black * chore: reformat for black bump --- pyproject.toml | 3 +-- src/s3_encryption/__init__.py | 1 + src/s3_encryption/materials/encrypted_data_key.py | 1 + src/s3_encryption/materials/materials.py | 1 + src/s3_encryption/metadata.py | 1 + src/s3_encryption/pipelines.py | 1 + 6 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 187b99a2..b05f22aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,8 +19,7 @@ test = [ "pytest>=8.4.1", ] dev = [ - # black >= 26 requires a reformat - "black>=24.3.0,<26.0.0", + "black>=24.3.0,<27.0.0", "ruff>=0.3.0", ] diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index adfb6886..46cdbdd1 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -1,6 +1,7 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 """Top-level S3 Encryption Client v3 for Python package.""" + import io from attrs import define, field diff --git a/src/s3_encryption/materials/encrypted_data_key.py b/src/s3_encryption/materials/encrypted_data_key.py index 28401b40..b2c2359a 100644 --- a/src/s3_encryption/materials/encrypted_data_key.py +++ b/src/s3_encryption/materials/encrypted_data_key.py @@ -5,6 +5,7 @@ This module provides the EncryptedDataKey class which represents an encrypted data key used in the S3 encryption process. """ + from attrs import define diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index 9f72ab91..3966e17c 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -6,6 +6,7 @@ which contain the cryptographic materials needed for S3 object encryption and decryption operations. """ + from typing import Any from attrs import define, field diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py index b4378990..f42feadb 100644 --- a/src/s3_encryption/metadata.py +++ b/src/s3_encryption/metadata.py @@ -5,6 +5,7 @@ This module provides classes and utilities for managing encryption metadata for S3 objects, including serialization and deserialization of metadata. """ + import json from typing import Any diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 6046fb3a..37093803 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -5,6 +5,7 @@ This module provides pipelines for encrypting objects before they are put into S3 and decrypting objects after they are retrieved from S3. """ + import base64 import os From 0cfd2186af3bc7bb978720e6f988498ea732b55c Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Tue, 3 Feb 2026 16:55:45 -0800 Subject: [PATCH 54/81] chore: add all other languages, latest major versions, many more tests to test-server (#134) --- .github/workflows/all-ci.yml | 47 + .github/workflows/duvet-test-server.yml | 104 + .github/workflows/lint.yml | 2 +- .github/workflows/main.yml | 28 - .github/workflows/python-integ.yml | 57 + .github/workflows/test-server.yml | 150 ++ .github/workflows/test.yml | 78 - .gitignore | 4 + .gitmodules | 48 + cdk/bin/cdk.ts | 7 + cdk/lib/cdk-stack.ts | 27 +- test-server/Makefile | 201 +- test-server/README.md | 32 +- .../.duvet/.gitignore | 3 + .../cpp-v2-transition-server/CMakeLists.txt | 39 + test-server/cpp-v2-transition-server/Makefile | 37 + .../cpp-v2-transition-server/README.md | 37 + .../cpp-v2-transition-server/aws-sdk-cpp | 1 + test-server/cpp-v2-transition-server/main.cpp | 748 ++++++++ test-server/cpp-v3-server/.duvet/.gitignore | 3 + test-server/cpp-v3-server/.duvet/config.toml | 45 + test-server/cpp-v3-server/CMakeLists.txt | 49 + test-server/cpp-v3-server/Makefile | 43 + test-server/cpp-v3-server/README.md | 37 + test-server/cpp-v3-server/aws-sdk-cpp | 1 + test-server/cpp-v3-server/compliance.txt | 119 ++ test-server/cpp-v3-server/main.cpp | 776 ++++++++ .../go-v3-transition-server/.duvet/.gitignore | 3 + .../.duvet/config.toml | 27 + test-server/go-v3-transition-server/Makefile | 39 + test-server/go-v3-transition-server/README.md | 23 + test-server/go-v3-transition-server/go.mod | 35 + test-server/go-v3-transition-server/go.sum | 45 + .../go-v3-transition-server/local-go-s3ec | 1 + test-server/go-v3-transition-server/main.go | 385 ++++ test-server/go-v4-server/.duvet/.gitignore | 3 + test-server/go-v4-server/.duvet/config.toml | 27 + test-server/go-v4-server/Makefile | 39 + test-server/go-v4-server/README.md | 23 + test-server/go-v4-server/go.mod | 35 + test-server/go-v4-server/go.sum | 44 + test-server/go-v4-server/local-go-s3ec | 1 + test-server/go-v4-server/main.go | 385 ++++ test-server/java-server/gradle.properties | 11 - .../s3/CreateClientOperationImpl.java | 109 -- .../encryption/s3/GetObjectOperationImpl.java | 72 - .../amazon/encryption/s3/MetadataUtils.java | 43 - .../encryption/s3/PutObjectOperationImpl.java | 55 - test-server/java-tests/README.md | 4 +- test-server/java-tests/build.gradle.kts | 28 + .../amazon/encryption/s3/CBCDecryptTests.java | 184 ++ .../s3/ExhaustiveRoundTripTests1_25.java | 232 +++ .../amazon/encryption/s3/GCMTestSuite.java | 258 +++ .../s3/InstructionFileFailures.java | 1136 +++++++++++ .../amazon/encryption/s3/KC_GCMTestSuite.java | 391 ++++ .../amazon/encryption/s3/RangedGetTests.java | 1687 +++++++++++++++++ .../amazon/encryption/s3/ReEncryptTests.java | 648 +++++++ .../amazon/encryption/s3/RoundTripTests.java | 561 ++++-- .../amazon/encryption/s3/TestUtils.java | 812 ++++++++ .../.duvet/.gitignore | 3 + .../.duvet/config.toml | 27 + .../java-v3-transition-server/.gitignore | 1 + .../java-v3-transition-server/Makefile | 42 + .../README.md | 6 +- .../build.gradle.kts | 9 +- .../gradle.properties | 24 + .../gradle/wrapper/gradle-wrapper.jar | Bin .../gradle/wrapper/gradle-wrapper.properties | 0 .../gradlew | 0 .../gradlew.bat | 184 +- .../license.txt | 0 .../java-v3-transition-server/s3ec-staging | 1 + .../settings.gradle.kts | 0 .../smithy-build.json | 0 .../java-v3-transition-server/specification | 1 + .../s3/CreateClientOperationImpl.java | 198 ++ .../encryption/s3/GetObjectOperationImpl.java | 88 + .../amazon/encryption/s3/MetadataUtils.java | 43 + .../encryption/s3/PutObjectOperationImpl.java | 55 + .../encryption/s3/ReEncryptOperationImpl.java | 183 ++ .../encryption/s3/S3ECJavaTestServer.java | 56 + test-server/java-v4-server/.duvet/.gitignore | 3 + test-server/java-v4-server/.duvet/config.toml | 27 + test-server/java-v4-server/.gitignore | 1 + test-server/java-v4-server/Makefile | 42 + test-server/java-v4-server/README.md | 23 + test-server/java-v4-server/build.gradle.kts | 60 + test-server/java-v4-server/gradle.properties | 24 + .../gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 43453 bytes .../gradle/wrapper/gradle-wrapper.properties | 7 + test-server/java-v4-server/gradlew | 249 +++ test-server/java-v4-server/gradlew.bat | 92 + test-server/java-v4-server/license.txt | 4 + test-server/java-v4-server/s3ec-staging | 1 + .../java-v4-server/settings.gradle.kts | 19 + test-server/java-v4-server/smithy-build.json | 11 + test-server/java-v4-server/specification | 1 + .../s3/CreateClientOperationImpl.java | 219 +++ .../encryption/s3/GetObjectOperationImpl.java | 86 + .../amazon/encryption/s3/MetadataUtils.java | 43 + .../encryption/s3/PutObjectOperationImpl.java | 52 + .../encryption/s3/ReEncryptOperationImpl.java | 183 ++ .../encryption/s3/S3ECJavaTestServer.java | 24 +- test-server/model/client.smithy | 44 +- test-server/model/object.smithy | 77 +- .../.duvet/.gitignore | 3 + .../.duvet/config.toml | 27 + .../net-v3-transition-server/.gitignore | 44 + .../Controllers/ClientController.cs | 133 ++ .../Controllers/ObjectController.cs | 105 + test-server/net-v3-transition-server/Makefile | 43 + .../Models/ClientRequest.cs | 55 + .../Models/ClientResponse.cs | 8 + .../Models/ErrorModels.cs | 17 + .../NetV3TransitionServer.csproj | 27 + .../net-v3-transition-server/Program.cs | 17 + .../net-v3-transition-server/README.md | 66 + .../Services/ClientCacheService.cs | 28 + .../s3ec-v3-transition-branch | 1 + test-server/net-v4-server/.duvet/.gitignore | 3 + test-server/net-v4-server/.duvet/config.toml | 27 + test-server/net-v4-server/.gitignore | 44 + .../Controllers/ClientController.cs | 144 ++ .../Controllers/ObjectController.cs | 105 + test-server/net-v4-server/Makefile | 45 + .../net-v4-server/Models/ClientRequest.cs | 55 + .../net-v4-server/Models/ClientResponse.cs | 8 + .../net-v4-server/Models/ErrorModels.cs | 17 + test-server/net-v4-server/NetV4Server.csproj | 28 + test-server/net-v4-server/Program.cs | 17 + test-server/net-v4-server/README.md | 72 + .../Services/ClientCacheService.cs | 28 + .../net-v4-server/s3ec-net-v4-improved | 1 + .../.duvet/.gitignore | 3 + .../.duvet/config.toml | 24 + .../php-v2-transition-server/.gitignore | 4 + test-server/php-v2-transition-server/Makefile | 39 + .../php-v2-transition-server/composer.json | 36 + .../php-v2-transition-server/local-php-sdk | 1 + .../php-v2-transition-server/src/client.php | 82 + .../php-v2-transition-server/src/errors.php | 42 + .../src/get_object.php | 104 + .../php-v2-transition-server/src/index.php | 295 +++ .../src/put_object.php | 79 + test-server/php-v3-server/.duvet/.gitignore | 3 + test-server/php-v3-server/.duvet/config.toml | 39 + test-server/php-v3-server/.gitignore | 4 + test-server/php-v3-server/Makefile | 39 + test-server/php-v3-server/README.md | 66 + .../compliance_exceptions/client.txt | 170 ++ .../content-metadata-strategy.txt | 34 + .../content-metadata.txt | 50 + .../compliance_exceptions/decryption.txt | 25 + .../compliance_exceptions/encryption.txt | 26 + test-server/php-v3-server/composer.json | 36 + test-server/php-v3-server/local-php-sdk | 1 + test-server/php-v3-server/src/client.php | 77 + test-server/php-v3-server/src/errors.php | 42 + test-server/php-v3-server/src/get_object.php | 108 ++ test-server/php-v3-server/src/index.php | 295 +++ test-server/php-v3-server/src/put_object.php | 82 + .../python-v3-server/.duvet/.gitignore | 3 + .../python-v3-server/.duvet/config.toml | 22 + .../.gitignore | 0 test-server/python-v3-server/Makefile | 42 + .../README.md | 0 .../poetry.lock | 0 .../pyproject.toml | 0 .../src/__init__.py | 0 .../src/main.py | 0 .../tests/__init__.py | 0 test-server/ruby-v2-server/.duvet/.gitignore | 3 + test-server/ruby-v2-server/.duvet/config.toml | 33 + test-server/ruby-v2-server/.gitignore | 2 + test-server/ruby-v2-server/Gemfile | 15 + test-server/ruby-v2-server/Gemfile.lock | 111 ++ test-server/ruby-v2-server/Makefile | 43 + test-server/ruby-v2-server/README.md | 73 + test-server/ruby-v2-server/app.rb | 241 +++ test-server/ruby-v2-server/config.ru | 3 + .../ruby-v2-server/lib/client_manager.rb | 109 ++ .../ruby-v2-server/lib/error_handlers.rb | 42 + test-server/ruby-v2-server/lib/logger.rb | 105 + .../ruby-v2-server/lib/metadata_utils.rb | 50 + test-server/ruby-v2-server/local-ruby-sdk | 1 + test-server/ruby-v3-server/.bundle/config | 2 + test-server/ruby-v3-server/.duvet/.gitignore | 3 + test-server/ruby-v3-server/.duvet/config.toml | 33 + test-server/ruby-v3-server/.gitignore | 2 + test-server/ruby-v3-server/Gemfile | 15 + test-server/ruby-v3-server/Gemfile.lock | 111 ++ test-server/ruby-v3-server/Makefile | 43 + test-server/ruby-v3-server/README.md | 74 + test-server/ruby-v3-server/app.rb | 241 +++ test-server/ruby-v3-server/config.ru | 3 + .../ruby-v3-server/lib/client_manager.rb | 134 ++ .../ruby-v3-server/lib/error_handlers.rb | 42 + test-server/ruby-v3-server/lib/logger.rb | 105 + .../ruby-v3-server/lib/metadata_utils.rb | 50 + test-server/ruby-v3-server/local-ruby-sdk | 1 + .../spec-compliance-dashboard/.gitignore | 1 + .../generate_compliance_dashboard.py | 1049 ++++++++++ .../templates/homepage_styles.css | 335 ++++ .../templates/homepage_template.html | 21 + .../templates/report_template.html | 276 +++ .../templates/styles.css | 387 ++++ .../templates/summary_stats_template.html | 63 + test-server/specification | 1 + 208 files changed, 18394 insertions(+), 752 deletions(-) create mode 100644 .github/workflows/all-ci.yml create mode 100644 .github/workflows/duvet-test-server.yml delete mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/python-integ.yml create mode 100644 .github/workflows/test-server.yml delete mode 100644 .github/workflows/test.yml create mode 100644 .gitmodules create mode 100644 cdk/bin/cdk.ts create mode 100644 test-server/cpp-v2-transition-server/.duvet/.gitignore create mode 100644 test-server/cpp-v2-transition-server/CMakeLists.txt create mode 100644 test-server/cpp-v2-transition-server/Makefile create mode 100644 test-server/cpp-v2-transition-server/README.md create mode 160000 test-server/cpp-v2-transition-server/aws-sdk-cpp create mode 100644 test-server/cpp-v2-transition-server/main.cpp create mode 100644 test-server/cpp-v3-server/.duvet/.gitignore create mode 100644 test-server/cpp-v3-server/.duvet/config.toml create mode 100644 test-server/cpp-v3-server/CMakeLists.txt create mode 100644 test-server/cpp-v3-server/Makefile create mode 100644 test-server/cpp-v3-server/README.md create mode 160000 test-server/cpp-v3-server/aws-sdk-cpp create mode 100644 test-server/cpp-v3-server/compliance.txt create mode 100644 test-server/cpp-v3-server/main.cpp create mode 100644 test-server/go-v3-transition-server/.duvet/.gitignore create mode 100644 test-server/go-v3-transition-server/.duvet/config.toml create mode 100644 test-server/go-v3-transition-server/Makefile create mode 100644 test-server/go-v3-transition-server/README.md create mode 100644 test-server/go-v3-transition-server/go.mod create mode 100644 test-server/go-v3-transition-server/go.sum create mode 160000 test-server/go-v3-transition-server/local-go-s3ec create mode 100644 test-server/go-v3-transition-server/main.go create mode 100644 test-server/go-v4-server/.duvet/.gitignore create mode 100644 test-server/go-v4-server/.duvet/config.toml create mode 100644 test-server/go-v4-server/Makefile create mode 100644 test-server/go-v4-server/README.md create mode 100644 test-server/go-v4-server/go.mod create mode 100644 test-server/go-v4-server/go.sum create mode 160000 test-server/go-v4-server/local-go-s3ec create mode 100644 test-server/go-v4-server/main.go delete mode 100644 test-server/java-server/gradle.properties delete mode 100644 test-server/java-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java delete mode 100644 test-server/java-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java delete mode 100644 test-server/java-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java delete mode 100644 test-server/java-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java create mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/CBCDecryptTests.java create mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/ExhaustiveRoundTripTests1_25.java create mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTestSuite.java create mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java create mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTestSuite.java create mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/RangedGetTests.java create mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/ReEncryptTests.java create mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java create mode 100644 test-server/java-v3-transition-server/.duvet/.gitignore create mode 100644 test-server/java-v3-transition-server/.duvet/config.toml create mode 100644 test-server/java-v3-transition-server/.gitignore create mode 100644 test-server/java-v3-transition-server/Makefile rename test-server/{java-server => java-v3-transition-server}/README.md (63%) rename test-server/{java-server => java-v3-transition-server}/build.gradle.kts (77%) create mode 100644 test-server/java-v3-transition-server/gradle.properties rename test-server/{java-server => java-v3-transition-server}/gradle/wrapper/gradle-wrapper.jar (100%) rename test-server/{java-server => java-v3-transition-server}/gradle/wrapper/gradle-wrapper.properties (100%) rename test-server/{java-server => java-v3-transition-server}/gradlew (100%) rename test-server/{java-server => java-v3-transition-server}/gradlew.bat (96%) rename test-server/{java-server => java-v3-transition-server}/license.txt (100%) create mode 160000 test-server/java-v3-transition-server/s3ec-staging rename test-server/{java-server => java-v3-transition-server}/settings.gradle.kts (100%) rename test-server/{java-server => java-v3-transition-server}/smithy-build.json (100%) create mode 120000 test-server/java-v3-transition-server/specification create mode 100644 test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java create mode 100644 test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java create mode 100644 test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java create mode 100644 test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java create mode 100644 test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/ReEncryptOperationImpl.java create mode 100644 test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java create mode 100644 test-server/java-v4-server/.duvet/.gitignore create mode 100644 test-server/java-v4-server/.duvet/config.toml create mode 100644 test-server/java-v4-server/.gitignore create mode 100644 test-server/java-v4-server/Makefile create mode 100644 test-server/java-v4-server/README.md create mode 100644 test-server/java-v4-server/build.gradle.kts create mode 100644 test-server/java-v4-server/gradle.properties create mode 100644 test-server/java-v4-server/gradle/wrapper/gradle-wrapper.jar create mode 100644 test-server/java-v4-server/gradle/wrapper/gradle-wrapper.properties create mode 100755 test-server/java-v4-server/gradlew create mode 100644 test-server/java-v4-server/gradlew.bat create mode 100644 test-server/java-v4-server/license.txt create mode 160000 test-server/java-v4-server/s3ec-staging create mode 100644 test-server/java-v4-server/settings.gradle.kts create mode 100644 test-server/java-v4-server/smithy-build.json create mode 120000 test-server/java-v4-server/specification create mode 100644 test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java create mode 100644 test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java create mode 100644 test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java create mode 100644 test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java create mode 100644 test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/ReEncryptOperationImpl.java rename test-server/{java-server => java-v4-server}/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java (67%) create mode 100644 test-server/net-v3-transition-server/.duvet/.gitignore create mode 100644 test-server/net-v3-transition-server/.duvet/config.toml create mode 100644 test-server/net-v3-transition-server/.gitignore create mode 100644 test-server/net-v3-transition-server/Controllers/ClientController.cs create mode 100644 test-server/net-v3-transition-server/Controllers/ObjectController.cs create mode 100644 test-server/net-v3-transition-server/Makefile create mode 100644 test-server/net-v3-transition-server/Models/ClientRequest.cs create mode 100644 test-server/net-v3-transition-server/Models/ClientResponse.cs create mode 100644 test-server/net-v3-transition-server/Models/ErrorModels.cs create mode 100644 test-server/net-v3-transition-server/NetV3TransitionServer.csproj create mode 100644 test-server/net-v3-transition-server/Program.cs create mode 100644 test-server/net-v3-transition-server/README.md create mode 100644 test-server/net-v3-transition-server/Services/ClientCacheService.cs create mode 160000 test-server/net-v3-transition-server/s3ec-v3-transition-branch create mode 100644 test-server/net-v4-server/.duvet/.gitignore create mode 100644 test-server/net-v4-server/.duvet/config.toml create mode 100644 test-server/net-v4-server/.gitignore create mode 100644 test-server/net-v4-server/Controllers/ClientController.cs create mode 100644 test-server/net-v4-server/Controllers/ObjectController.cs create mode 100644 test-server/net-v4-server/Makefile create mode 100644 test-server/net-v4-server/Models/ClientRequest.cs create mode 100644 test-server/net-v4-server/Models/ClientResponse.cs create mode 100644 test-server/net-v4-server/Models/ErrorModels.cs create mode 100644 test-server/net-v4-server/NetV4Server.csproj create mode 100644 test-server/net-v4-server/Program.cs create mode 100644 test-server/net-v4-server/README.md create mode 100644 test-server/net-v4-server/Services/ClientCacheService.cs create mode 160000 test-server/net-v4-server/s3ec-net-v4-improved create mode 100644 test-server/php-v2-transition-server/.duvet/.gitignore create mode 100644 test-server/php-v2-transition-server/.duvet/config.toml create mode 100644 test-server/php-v2-transition-server/.gitignore create mode 100644 test-server/php-v2-transition-server/Makefile create mode 100644 test-server/php-v2-transition-server/composer.json create mode 120000 test-server/php-v2-transition-server/local-php-sdk create mode 100644 test-server/php-v2-transition-server/src/client.php create mode 100644 test-server/php-v2-transition-server/src/errors.php create mode 100644 test-server/php-v2-transition-server/src/get_object.php create mode 100644 test-server/php-v2-transition-server/src/index.php create mode 100644 test-server/php-v2-transition-server/src/put_object.php create mode 100644 test-server/php-v3-server/.duvet/.gitignore create mode 100644 test-server/php-v3-server/.duvet/config.toml create mode 100644 test-server/php-v3-server/.gitignore create mode 100644 test-server/php-v3-server/Makefile create mode 100644 test-server/php-v3-server/README.md create mode 100644 test-server/php-v3-server/compliance_exceptions/client.txt create mode 100644 test-server/php-v3-server/compliance_exceptions/content-metadata-strategy.txt create mode 100644 test-server/php-v3-server/compliance_exceptions/content-metadata.txt create mode 100644 test-server/php-v3-server/compliance_exceptions/decryption.txt create mode 100644 test-server/php-v3-server/compliance_exceptions/encryption.txt create mode 100644 test-server/php-v3-server/composer.json create mode 160000 test-server/php-v3-server/local-php-sdk create mode 100644 test-server/php-v3-server/src/client.php create mode 100644 test-server/php-v3-server/src/errors.php create mode 100644 test-server/php-v3-server/src/get_object.php create mode 100644 test-server/php-v3-server/src/index.php create mode 100644 test-server/php-v3-server/src/put_object.php create mode 100644 test-server/python-v3-server/.duvet/.gitignore create mode 100644 test-server/python-v3-server/.duvet/config.toml rename test-server/{python-server => python-v3-server}/.gitignore (100%) create mode 100644 test-server/python-v3-server/Makefile rename test-server/{python-server => python-v3-server}/README.md (100%) rename test-server/{python-server => python-v3-server}/poetry.lock (100%) rename test-server/{python-server => python-v3-server}/pyproject.toml (100%) rename test-server/{python-server => python-v3-server}/src/__init__.py (100%) rename test-server/{python-server => python-v3-server}/src/main.py (100%) rename test-server/{python-server => python-v3-server}/tests/__init__.py (100%) create mode 100644 test-server/ruby-v2-server/.duvet/.gitignore create mode 100644 test-server/ruby-v2-server/.duvet/config.toml create mode 100644 test-server/ruby-v2-server/.gitignore create mode 100644 test-server/ruby-v2-server/Gemfile create mode 100644 test-server/ruby-v2-server/Gemfile.lock create mode 100644 test-server/ruby-v2-server/Makefile create mode 100644 test-server/ruby-v2-server/README.md create mode 100644 test-server/ruby-v2-server/app.rb create mode 100644 test-server/ruby-v2-server/config.ru create mode 100644 test-server/ruby-v2-server/lib/client_manager.rb create mode 100644 test-server/ruby-v2-server/lib/error_handlers.rb create mode 100644 test-server/ruby-v2-server/lib/logger.rb create mode 100644 test-server/ruby-v2-server/lib/metadata_utils.rb create mode 160000 test-server/ruby-v2-server/local-ruby-sdk create mode 100644 test-server/ruby-v3-server/.bundle/config create mode 100644 test-server/ruby-v3-server/.duvet/.gitignore create mode 100644 test-server/ruby-v3-server/.duvet/config.toml create mode 100644 test-server/ruby-v3-server/.gitignore create mode 100644 test-server/ruby-v3-server/Gemfile create mode 100644 test-server/ruby-v3-server/Gemfile.lock create mode 100644 test-server/ruby-v3-server/Makefile create mode 100644 test-server/ruby-v3-server/README.md create mode 100644 test-server/ruby-v3-server/app.rb create mode 100644 test-server/ruby-v3-server/config.ru create mode 100644 test-server/ruby-v3-server/lib/client_manager.rb create mode 100644 test-server/ruby-v3-server/lib/error_handlers.rb create mode 100644 test-server/ruby-v3-server/lib/logger.rb create mode 100644 test-server/ruby-v3-server/lib/metadata_utils.rb create mode 160000 test-server/ruby-v3-server/local-ruby-sdk create mode 100644 test-server/spec-compliance-dashboard/.gitignore create mode 100644 test-server/spec-compliance-dashboard/generate_compliance_dashboard.py create mode 100644 test-server/spec-compliance-dashboard/templates/homepage_styles.css create mode 100644 test-server/spec-compliance-dashboard/templates/homepage_template.html create mode 100644 test-server/spec-compliance-dashboard/templates/report_template.html create mode 100644 test-server/spec-compliance-dashboard/templates/styles.css create mode 100644 test-server/spec-compliance-dashboard/templates/summary_stats_template.html create mode 160000 test-server/specification diff --git a/.github/workflows/all-ci.yml b/.github/workflows/all-ci.yml new file mode 100644 index 00000000..c65364b7 --- /dev/null +++ b/.github/workflows/all-ci.yml @@ -0,0 +1,47 @@ +name: All CI + +on: + push: + branches: [ main, staging ] + pull_request: + workflow_dispatch: + inputs: + python-version: + description: 'Python version to use' + default: '3.11' + required: false + type: string + +jobs: + python-lint: + name: Lint + uses: ./.github/workflows/lint.yml + + run-test-server: + permissions: + id-token: write + contents: read + name: Run TestServer Tests + uses: ./.github/workflows/test-server.yml + with: + python-version: ${{ inputs.python-version || '3.11' }} + secrets: inherit + + python-integ: + permissions: + id-token: write + contents: read + name: Python Integration Tests + uses: ./.github/workflows/python-integ.yml + with: + python-version: ${{ inputs.python-version || '3.11' }} + secrets: inherit + + run-duvet-test-server: + permissions: + id-token: write + contents: read + pages: write + name: Run Duvet + uses: ./.github/workflows/duvet-test-server.yml + secrets: inherit diff --git a/.github/workflows/duvet-test-server.yml b/.github/workflows/duvet-test-server.yml new file mode 100644 index 00000000..f4bac5a8 --- /dev/null +++ b/.github/workflows/duvet-test-server.yml @@ -0,0 +1,104 @@ +name: Generate Duvet Report for TestServer + +on: + workflow_call: + # Optional inputs that can be provided when calling this workflow + +jobs: + duvet: + runs-on: macos-latest + permissions: + id-token: write + contents: read + pages: write + + steps: + - name: Checkout code + uses: actions/checkout@v5 + with: + submodules: true + token: ${{ secrets.PAT_FOR_SPEC }} + + - name: Checkout CPP code cpp-v3 + uses: actions/checkout@v5 + with: + submodules: recursive + repository: aws/aws-sdk-cpp + ref: main + path: test-server/cpp-v3-server/aws-sdk-cpp/ + + - name: Setup Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: stable + + - name: Clone duvet repository + run: git clone https://github.com/awslabs/duvet.git /tmp/duvet + + - name: Build and install duvet + run: | + cd /tmp/duvet + cargo xtask build + cargo install --path ./duvet + + - name: Run duvet + if: always() + run: cd test-server && make duvet + + - name: Upload duvet reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: reports + include-hidden-files: true + path: test-server/*-server/.duvet/reports/report.html + + - name: Generate compliance dashboard + if: always() + run: | + cd test-server/spec-compliance-dashboard + python generate_compliance_dashboard.py + + - name: Create dashboard redirect index.html + if: always() + run: | + cat > test-server/index.html << 'EOF' + + + + + + Redirecting to Compliance Dashboard... + + +

Redirecting to Compliance Dashboard...

+ + + EOF + + - name: Upload compliance dashboard + if: always() + uses: actions/upload-artifact@v4 + with: + name: compliance-dashboard + include-hidden-files: true + path: | + test-server/spec-compliance-dashboard/compliance_homepage.html + test-server/*/compliance_summary_report.html + test-server/*/.duvet/reports/report.html + test-server/spec-compliance-dashboard/templates/* + test-server/index.html + + - name: Setup Pages + if: always() && (github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/fireegg-test-servers') && github.event_name == 'push' + uses: actions/configure-pages@v5 + + - name: Upload Pages artifact + if: always() && (github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/fireegg-test-servers') && github.event_name == 'push' + uses: actions/upload-pages-artifact@v3 + with: + path: test-server/ + + - name: Deploy to GitHub Pages + if: always() && (github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/fireegg-test-servers') && github.event_name == 'push' + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 16057711..bb1655bb 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -8,7 +8,7 @@ on: jobs: lint: - runs-on: ubuntu-latest + runs-on: macos-15 steps: - name: Checkout code diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 675a0099..00000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Main Workflow - -on: - push: - branches: [ staging ] - pull_request: - workflow_dispatch: - inputs: - python-version: - description: 'Python version to use' - default: '3.11' - required: false - type: string - -jobs: - lint: - name: Lint - uses: ./.github/workflows/lint.yml - - run-tests: - name: Run Tests - uses: ./.github/workflows/test.yml - with: - python-version: ${{ inputs.python-version || '3.11' }} - permissions: - id-token: write - contents: read - secrets: inherit diff --git a/.github/workflows/python-integ.yml b/.github/workflows/python-integ.yml new file mode 100644 index 00000000..9e5ae818 --- /dev/null +++ b/.github/workflows/python-integ.yml @@ -0,0 +1,57 @@ +name: Python Integration Tests + +on: + workflow_call: + inputs: + python-version: + description: "Python version to use" + default: "3.11" + required: false + type: string + +jobs: + python-integ: + runs-on: macos-14-large + permissions: + id-token: write + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v5 + with: + submodules: false + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version || '3.11' }} + + - name: Cache uv dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/uv + key: ${{ runner.os }}-uv-${{ hashFiles('./test-server/python-v3-server/**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-uv- + + - name: Install Uv + run: pip install uv + + - name: Install dependencies + run: make install + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role + aws-region: us-west-2 + + - name: Run unit tests + run: make test-unit + + - name: Run integration tests + run: make test-integration + env: + CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} + CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml new file mode 100644 index 00000000..4fa10666 --- /dev/null +++ b/.github/workflows/test-server.yml @@ -0,0 +1,150 @@ +name: Run TestServer Tests + +on: + workflow_call: + # Optional inputs that can be provided when calling this workflow + inputs: + python-version: + description: "Python version to use" + default: "3.11" + required: false + type: string + +jobs: + test-server: + runs-on: macos-14-large + permissions: + id-token: write + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v5 + with: + submodules: false + token: ${{ secrets.PAT_FOR_SPEC }} + + # There are a lot of submodules here + # This initializes the checkouts in parallel (--jobs) + # rather than in series the way actions/checkout@v5 does it. + + - name: Get CPU count + id: cpu-count + run: echo "count=$(node -p 'require("os").cpus().length')" >> $GITHUB_OUTPUT + + - name: Setup git submodules with PAT + run: | + git config --global url."https://github.com/".insteadOf "git@github.com:" + git config --global credential.helper store + echo "https://x-token-auth:${{ secrets.PAT_FOR_SPEC }}@github.com" > ~/.git-credentials + + - name: Optimize git for performance + run: | + git config --global fetch.parallel ${{ steps.cpu-count.outputs.count }} + git config --global submodule.fetchJobs ${{ steps.cpu-count.outputs.count }} + git config --global remote.origin.tagOpt --no-tags + + - name: Checkout submodules with --jobs + run: | + git submodule update --init --depth 1 --single-branch --jobs ${{ steps.cpu-count.outputs.count }} + + - name: Update cpp submodules recursively with --jobs + run: | + git submodule update --init --recursive \ + --depth 1 --single-branch \ + --jobs ${{ steps.cpu-count.outputs.count }} \ + --force \ + test-server/cpp-v2-transition-server/aws-sdk-cpp \ + test-server/cpp-v3-server/aws-sdk-cpp + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.4.7" + bundler-cache: true + + - name: Set up PHP with Composer + uses: shivammathur/setup-php@verbose + with: + php-version: "8.1" + + - name: Install PHP V2 Transition dependencies + working-directory: ./test-server/php-v2-transition-server + shell: bash + run: composer install + + - name: Install PHP V3 dependencies + working-directory: ./test-server/php-v3-server + shell: bash + run: composer install + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.25 + + - name: Install C++ dependencies + run: | + brew install libmicrohttpd nlohmann-json ossp-uuid + + # Cache Gradle dependencies and build outputs + - name: Cache Gradle packages + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + test-server/java-tests/.gradle + key: ${{ runner.os }}-gradle-${{ hashFiles('test-server/java-tests/**/gradle-wrapper.properties', 'test-server/java-tests/**/*.gradle*') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role + aws-region: us-west-2 + + - name: Build the servers + run: cd test-server && make build-all-servers + env: + MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }} + AWS_REGION: us-west-2 + + - name: Start the servers + run: cd test-server && make start-all-servers + env: + AWS_REGION: us-west-2 + TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} + TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} + + - name: Wait for servers to start + run: cd test-server && make wait-all-servers + env: + MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }} + + - name: Run run-tests + run: cd test-server && make test-servers-run-tests + env: + AWS_REGION: us-west-2 + TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} + TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} + GRADLE_OPTS: "-Dorg.gradle.daemon=true -Dorg.gradle.parallel=true -Dorg.gradle.caching=true" + + - name: Upload server logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: server-logs + path: | + test-server/*/server.log + + - name: Stop the servers + run: cd test-server && make test-servers-stop + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: results + path: test-server/java-tests/build/reports/tests/integ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index f8025246..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Run Tests - -on: - workflow_call: - # Optional inputs that can be provided when calling this workflow - inputs: - python-version: - description: 'Python version to use' - default: '3.11' - required: false - type: string - -jobs: - test: - runs-on: ubuntu-latest - permissions: - id-token: write - contents: read - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python-version || '3.11' }} - - # Cache uv dependencies - - name: Cache uv dependencies - uses: actions/cache@v3 - with: - path: ~/.cache/uv - key: ${{ runner.os }}-uv-${{ hashFiles('**/pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-uv- - - - name: Install Uv - run: pip install uv - - # Cache Gradle dependencies and build outputs - - name: Cache Gradle packages - uses: actions/cache@v3 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - test-server/java-server/.gradle - test-server/java-tests/.gradle - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - - name: Install dependencies - run: make install - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role - aws-region: us-west-2 - - - name: Run unit tests - run: make test-unit - - - name: Run integration tests - run: make test-integration - env: - CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} - CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} - - - name: Run test-server tests - run: cd test-server && make ci - env: - AWS_REGION: us-west-2 - TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} - TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - GRADLE_OPTS: "-Dorg.gradle.daemon=true -Dorg.gradle.parallel=true -Dorg.gradle.caching=true" diff --git a/.gitignore b/.gitignore index 0e29a9fb..5cd8f239 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ __pycache__/ # Distribution / packaging dist/ build/ +bin/ *.egg-info/ # Uv @@ -51,3 +52,6 @@ gradle-app.setting .DS_Store smithy-java-core/out + +# test server +*.pid diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..75e91f99 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,48 @@ +[submodule "test-server/ruby-v2-server/local-ruby-sdk"] + path = test-server/ruby-v2-server/local-ruby-sdk + url = git@github.com:aws/aws-sdk-ruby.git + branch = version-3 +[submodule "test-server/ruby-v3-server/local-ruby-sdk"] + path = test-server/ruby-v3-server/local-ruby-sdk + url = git@github.com:aws/aws-sdk-ruby.git + branch = version-3 +[submodule "test-server/php-v3-server/local-php-sdk"] + path = test-server/php-v3-server/local-php-sdk + url = git@github.com:aws/aws-sdk-php.git + branch = master +[submodule "test-server/go-v4-server/local-go-s3ec"] + path = test-server/go-v4-server/local-go-s3ec + url = https://github.com/aws/amazon-s3-encryption-client-go + branch = main +[submodule "test-server/java-v3-transition-server/s3ec-staging"] + path = test-server/java-v3-transition-server/s3ec-staging + url = git@github.com:aws/amazon-s3-encryption-client-java.git + branch = main-3.x +[submodule "test-server/java-v4-server/s3ec-staging"] + path = test-server/java-v4-server/s3ec-staging + url = git@github.com:aws/amazon-s3-encryption-client-java.git + branch = main +[submodule "test-server/specification"] + path = test-server/specification + url = git@github.com:awslabs/private-aws-encryption-sdk-specification-staging.git + branch = fire-egg-staging +[submodule "test-server/net-v4-server/s3ec-net-v4-improved"] + path = test-server/net-v4-server/s3ec-net-v4-improved + url = https://github.com/aws/amazon-s3-encryption-client-dotnet.git + branch = main +[submodule "test-server/go-v3-transition-server/local-go-s3ec"] + path = test-server/go-v3-transition-server/local-go-s3ec + url = https://github.com/aws/amazon-s3-encryption-client-go + branch = main +[submodule "test-server/net-v3-transition-server/s3ec-v3-transition-branch"] + path = test-server/net-v3-transition-server/s3ec-v3-transition-branch + url = https://github.com/aws/amazon-s3-encryption-client-dotnet.git + branch = v4sdk-development +[submodule "test-server/cpp-v2-transition-server/aws-sdk-cpp"] + path = test-server/cpp-v2-transition-server/aws-sdk-cpp + url = git@github.com:aws/aws-sdk-cpp.git + branch = main +[submodule "test-server/cpp-v3-server/aws-sdk-cpp"] + path = test-server/cpp-v3-server/aws-sdk-cpp + url = git@github.com:aws/aws-sdk-cpp.git + branch = main diff --git a/cdk/bin/cdk.ts b/cdk/bin/cdk.ts new file mode 100644 index 00000000..08214db5 --- /dev/null +++ b/cdk/bin/cdk.ts @@ -0,0 +1,7 @@ +#!/usr/bin/env node +import 'source-map-support/register'; +import * as cdk from 'aws-cdk-lib'; +import { S3ECPythonGithub } from '../lib/cdk-stack'; + +const app = new cdk.App(); +new S3ECPythonGithub(app, 'S3ECPythonGithub'); diff --git a/cdk/lib/cdk-stack.ts b/cdk/lib/cdk-stack.ts index cdb7c489..1fad4b74 100644 --- a/cdk/lib/cdk-stack.ts +++ b/cdk/lib/cdk-stack.ts @@ -10,6 +10,8 @@ import { PolicyDocument, PolicyStatement, FederatedPrincipal, + ArnPrincipal, + CompositePrincipal, ManagedPolicy, } from "aws-cdk-lib/aws-iam"; import { @@ -99,23 +101,31 @@ export class S3ECPythonGithub extends cdk.Stack { new PolicyStatement({ effect: Effect.ALLOW, actions: [ + "s3:HeadObject", // Only get object metadata "s3:PutObject", "s3:GetObject", "s3:DeleteObject", + "s3:DeleteObjectVersion" // For S3EC-NET repo ], resources: [ S3ECGithubTestS3Bucket.bucketArn + "/*", // object-level permissions need this extra path S3ECTestServerGithubBucket.bucketArn + "/*", // Add permissions for the new test-server bucket + "arn:aws:s3:::aws-net-sdk-*/*" // permission for object inside S3EC .net bucket. For S3EC-NET repo ], }), new PolicyStatement({ effect: Effect.ALLOW, actions: [ + "s3:CreateBucket", // For S3EC-NET repo + "s3:DeleteBucket", // For S3EC-NET repo "s3:ListBucket", + "s3:ListBucketVersions", // For S3EC-NET repo + "s3:GetBucketAcl" // For S3EC-NET repo ], resources: [ S3ECGithubTestS3Bucket.bucketArn, S3ECTestServerGithubBucket.bucketArn, // Add permissions for the new test-server bucket + "arn:aws:s3:::aws-net-sdk-*", // permission for S3EC .net bucket. For S3EC-NET repo ], }), ] @@ -155,16 +165,29 @@ export class S3ECPythonGithub extends cdk.Stack { "token.actions.githubusercontent.com:aud": "sts.amazonaws.com" }, "StringLike": { - "token.actions.githubusercontent.com:sub": "repo:aws/amazon-s3-encryption-client-python:*" + "token.actions.githubusercontent.com:sub": [ + "repo:aws/amazon-s3-encryption-client-python:*", + "repo:aws/private-amazon-s3-encryption-client-dotnet-staging:*" // For S3EC-NET repo + ] } }, "sts:AssumeRoleWithWebIdentity" ) + + // ToolsDevelopment role principal + const ToolsDevelopmentPrincipal = new ArnPrincipal("arn:aws:iam::" + this.account + ":role/ToolsDevelopment") + + // Composite principal to allow both GitHub Actions and ToolsDevelopment to assume the role + const CompositePrincipalForRole = new CompositePrincipal( + GithubActionsPrincipal, + ToolsDevelopmentPrincipal + ) + const S3ECGithubTestRole = new Role( this, "s3-github-test-role", { - assumedBy: GithubActionsPrincipal, + assumedBy: CompositePrincipalForRole, roleName: "S3EC-Python-Github-test-role", description: " Grant GitHub S3 put and get and KMS encrypt, decrypt, and generate access for testing", path: "/", diff --git a/test-server/Makefile b/test-server/Makefile index afbe97b5..21b5c98b 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -1,63 +1,81 @@ # Makefile for S3 Encryption Client Testing -.PHONY: all start-servers start-python-server start-java-server run-tests stop-servers clean ci check-env help - -# Default target -all: start-servers run-tests +.PHONY: test-servers-all test-servers-start test-servers-run-tests test-servers-stop test-servers-clean test-servers-ci test-servers-check-env test-servers-help # CI target for GitHub Actions -ci: start-servers run-tests stop-servers +test-servers-ci: + $(MAKE) build-all-servers + $(MAKE) start-all-servers + $(MAKE) wait-all-servers + $(MAKE) test-servers-run-tests + $(MAKE) test-servers-stop +SERVER_DIRS := $(shell find . -maxdepth 1 -type d -name '*-server' | sed 's|^\./||' | $(if $(FILTER),grep -E "$$(echo '$(FILTER)' | sed 's/,/|/g')",cat) | sort) -# Start Python server in background -start-python-server: - @echo "Starting Python server..." - cd python-server && \ - python -m venv .venv && \ - .venv/bin/python -m ensurepip && \ - .venv/bin/python -m pip install -e . && \ - .venv/bin/python -m pip install -e ../.. && \ - AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ - AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ - AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ - AWS_REGION="us-west-2" \ - .venv/bin/python src/main.py & echo $$! > ../python-server.pid - @echo "Python server starting..." +BUILD_SERVER_TARGETS := $(addprefix build-, $(SERVER_DIRS)) +START_SERVER_TARGETS := $(addprefix start-, $(SERVER_DIRS)) +WAIT_SERVER_TARGETS := $(addprefix wait-, $(SERVER_DIRS)) -# Start Java server in background -start-java-server: - @echo "Starting Java server..." - cd java-server && \ - AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ - AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ - AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ - AWS_REGION="us-west-2" \ - ./gradlew --build-cache --parallel run & echo $$! > ../java-server.pid - @echo "Java server starting..." - -# Start both servers in parallel -start-servers: - @echo "Starting servers in parallel..." - @$(MAKE) -j2 start-python-server start-java-server - @echo "Waiting for servers to be ready..." - @for i in $$(seq 1 360); do \ - if nc -z localhost 8080 && nc -z localhost 8081; then \ - echo "Ports are open, waiting for servers to initialize..."; \ - sleep 5; \ - echo "Both servers are ready!"; \ - break; \ - fi; \ - if [ $$i -eq 360 ]; then \ - echo "Timeout waiting for servers to start"; \ - exit 1; \ - fi; \ - echo "Waiting for servers to start ($$i/360)..."; \ - sleep 1; \ +# Build all servers in parallel +build-all-servers: + @echo "[`date +%H:%M:%S`] Building all servers..." + @$(MAKE) $(BUILD_SERVER_TARGETS) + @echo "[`date +%H:%M:%S`] All servers built." + @echo "Stopping dotnet servers... this is here to speed up CI because if this target is run with -j it will stay open until it shutdown." + @dotnet build-server shutdown + @echo "[`date +%H:%M:%S`] Dotnet build servers stopped" + +$(BUILD_SERVER_TARGETS): build-%: + @if [ -f $*/Makefile ]; then \ + echo "[`date +%H:%M:%S`] Building server in $*..." && \ + $(MAKE) -C $* build-server && \ + echo "[`date +%H:%M:%S`] Server $* built successfully"; \ + else \ + echo "❌ Error: no Makefile found in $*"; \ + exit 1; \ + fi + +# Build and start all servers +test-servers-start: + @echo "Building all servers..." + $(MAKE) build-all-servers + @echo "Starting all servers..." + $(MAKE) start-all-servers + @echo "Waiting for servers to start..." + @for dir in $(SERVER_DIRS); do \ + echo "Waiting for server in $$dir..."; \ + $(MAKE) -C $$dir wait-for-server; \ done +start-all-servers: + @$(MAKE) $(START_SERVER_TARGETS) + +$(START_SERVER_TARGETS): start-%: + @if [ -f $*/Makefile ]; then \ + echo "Starting server in $*..." && \ + $(MAKE) -C $* start-server; \ + else \ + echo "❌ Error: no Makefile found in $*"; \ + exit 1; \ + fi + +wait-all-servers: + @echo "Waiting for all servers to be ready..." + $(MAKE) $(WAIT_SERVER_TARGETS) + @echo "All servers are ready!" + +$(WAIT_SERVER_TARGETS): wait-%: + @if [ -f $*/Makefile ]; then \ + echo "Waiting server in $*..." && \ + $(MAKE) -C $* wait-for-server; \ + else \ + echo "❌ Error: no Makefile found in $*"; \ + exit 1; \ + fi + # Run the Java tests -run-tests: +test-servers-run-tests: @echo "Running Java tests..." @echo "Exporting environment variables from servers to tests..." @# Extract AWS environment variables from the current shell and pass them to the tests @@ -66,46 +84,75 @@ run-tests: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./gradlew --build-cache --parallel integ + ./gradlew --build-cache --info --parallel --no-daemon integ \ + $(if $(TEST),--tests "$(TEST)",) \ + -Dtest.filter.servers="$(FILTER)" @echo "Tests completed successfully" # Stop the servers -stop-servers: +test-servers-stop: @echo "Stopping servers..." - @if [ -f python-server.pid ]; then \ - kill $$(cat python-server.pid) 2>/dev/null || true; \ - rm python-server.pid; \ - fi - @if [ -f java-server.pid ]; then \ - kill $$(cat java-server.pid) 2>/dev/null || true; \ - rm java-server.pid; \ - fi + @for dir in $(SERVER_DIRS); do \ + echo "Stopping server in $$dir..."; \ + $(MAKE) -C $$dir stop-server; \ + done @echo "Servers stopped" -# Clean up logs and pid files -clean: stop-servers - @echo "Cleaning up..." - @rm -f python-server.log java-server.log - @echo "Cleanup complete" - # Help target -help: +test-servers-help: @echo "Available targets:" - @echo " all : Start servers and run tests (default, output to stdout)" - @echo " ci : Run in CI mode (start servers, run tests, stop servers)" - @echo " start-servers : Start Python and Java servers in parallel (output to stdout)" - @echo " start-python-server: Start only the Python server" - @echo " start-java-server : Start only the Java server" - @echo " run-tests : Run Java tests" - @echo " stop-servers : Stop running servers" - @echo " clean : Stop servers and clean up logs" - @echo " check-env : Check if required environment variables are set" - @echo " help : Show this help message" + @echo " test-servers-all : Start servers and run tests (default, output to stdout)" + @echo " test-servers-ci : Run in CI mode (start servers, run tests, stop servers)" + @echo " test-servers-start : Start all servers in parallel" + @echo " test-servers-run-tests : Run Java tests" + @echo " test-servers-stop : Stop running servers" + @echo " test-servers-check-env : Check if required environment variables are set" + @echo " test-servers-help : Show this help message" # Check if required environment variables are set -check-env: +test-servers-check-env: @echo "Checking required environment variables..." @if [ -z "$$AWS_ACCESS_KEY_ID" ]; then echo "AWS_ACCESS_KEY_ID is not set"; else echo "AWS_ACCESS_KEY_ID is set"; fi @if [ -z "$$AWS_SECRET_ACCESS_KEY" ]; then echo "AWS_SECRET_ACCESS_KEY is not set"; else echo "AWS_SECRET_ACCESS_KEY is set"; fi @if [ -z "$$AWS_SESSION_TOKEN" ]; then echo "AWS_SESSION_TOKEN is not set"; else echo "AWS_SESSION_TOKEN is set"; fi @if [ -z "$$AWS_REGION" ]; then echo "AWS_REGION is not set (will use us-west-2 as default)"; else echo "AWS_REGION is set to $$AWS_REGION"; fi + +TIMEOUT := 360 + +wait-for-port: + @if [ -z "$(PORT)" ]; then \ + echo "❌ Error: PORT is required"; \ + exit 1; \ + fi + @echo "Starting to wait for $$PORT to start"; + @for i in $$(seq 1 $(TIMEOUT)); do \ + if nc -z localhost $$PORT; then \ + echo "Ports are open, waiting for servers to initialize..."; \ + sleep 5; \ + echo "Server at $$PORT is ready!"; \ + break; \ + fi; \ + if [ $$i -eq $(TIMEOUT) ]; then \ + echo "Timeout waiting for $$PORT start"; \ + exit 1; \ + fi; \ + echo "Waiting for $$PORT to start ($$i/$(TIMEOUT))..."; \ + sleep 3; \ + done + +# Here are some helpful curl commands +# that you can use to test specific test servers: +test-create-client: + @echo $(PORT) + @curl -X POST \ + -H "Content-Type: application/json" \ + -H "User-Agent: smithy-java/0.0.3 ua/2.1 os/macos#15.5 lang/java#23.0.2" \ + -d '{"config":{"enableLegacyUnauthenticatedModes":false,"enableDelayedAuthenticationMode":false,"enableLegacyWrappingAlgorithms":false,"keyMaterial":{"kmsKeyId":"arn:aws:kms:us-west-2:370957321024:alias/S3EC-Test-Server-Github-KMS-Key"}}}' \ + http://localhost:$(PORT)/client + +duvet: + @echo "Running duvet reports..." + @for dir in $(SERVER_DIRS); do \ + echo "Running make duvet in $$dir..."; \ + $(MAKE) -C $$dir duvet; \ + done diff --git a/test-server/README.md b/test-server/README.md index a320d1d1..48187fc3 100644 --- a/test-server/README.md +++ b/test-server/README.md @@ -28,11 +28,8 @@ make ci # Start Python and Java servers in parallel make start-servers -# Start only the Python server -make start-python-server - -# Start only the Java server -make start-java-server +# Start only the Python S3EC V3 server +make start-python-v3-server # Run Java tests make run-tests @@ -59,3 +56,28 @@ Performance optimizations have been implemented to speed up the test-server CI p - JVM optimizations For detailed information about the optimizations, see [OPTIMIZATION.md](./OPTIMIZATION.md). + +### Duvet + +To check duvet you need to install Rust. +Until the latest version of Duvet is release + +```bash + git clone https://github.com/awslabs/duvet.git /tmp/duvet + pushd /tmp/duvet + cargo xtask build + cargo install --path ./duvet + popd rm -rf /tmp/duvet +``` + +Inside each test server directory there is a `.duvet` directory that contains a `config.toml`. +This is the best way to configure `duvet`. + +You can adjust the source pattern or comment style as needed. +Examples: + +- `ruby-v2-server/.duvet/config.toml` + +There are Makefile targets, +but you can just run `make duvet` or `duvet report` inside a server directory to run the report. +To view the report `make view-report-mac` or `open .duvet/reports/report.html` diff --git a/test-server/cpp-v2-transition-server/.duvet/.gitignore b/test-server/cpp-v2-transition-server/.duvet/.gitignore new file mode 100644 index 00000000..93956e36 --- /dev/null +++ b/test-server/cpp-v2-transition-server/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ \ No newline at end of file diff --git a/test-server/cpp-v2-transition-server/CMakeLists.txt b/test-server/cpp-v2-transition-server/CMakeLists.txt new file mode 100644 index 00000000..b282dbc4 --- /dev/null +++ b/test-server/cpp-v2-transition-server/CMakeLists.txt @@ -0,0 +1,39 @@ +cmake_minimum_required(VERSION 3.16) +project(s3ec-cpp-v2-server) + +set(CMAKE_CXX_STANDARD 17) + +# Configure AWS SDK build options +set(BUILD_ONLY "kms;s3;s3-encryption" CACHE STRING "Build only KMS, S3, and S3-encryption components") +set(ENABLE_TESTING OFF CACHE BOOL "Disable testing") +set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build static libraries") + +# Add AWS SDK as subdirectory +add_subdirectory(aws-sdk-cpp) + +find_package(PkgConfig REQUIRED) +pkg_check_modules(LIBMICROHTTPD REQUIRED libmicrohttpd) + +find_package(nlohmann_json REQUIRED) + +add_executable(s3ec-server main.cpp) + +target_include_directories(s3ec-server PRIVATE + ${LIBMICROHTTPD_INCLUDE_DIRS} + /opt/homebrew/include +) + +target_link_directories(s3ec-server PRIVATE + ${LIBMICROHTTPD_LIBRARY_DIRS} + /opt/homebrew/lib +) + +target_link_libraries(s3ec-server + ${LIBMICROHTTPD_LIBRARIES} + aws-cpp-sdk-core + aws-cpp-sdk-kms + aws-cpp-sdk-s3 + aws-cpp-sdk-s3-encryption + nlohmann_json::nlohmann_json + uuid +) \ No newline at end of file diff --git a/test-server/cpp-v2-transition-server/Makefile b/test-server/cpp-v2-transition-server/Makefile new file mode 100644 index 00000000..0383b4d8 --- /dev/null +++ b/test-server/cpp-v2-transition-server/Makefile @@ -0,0 +1,37 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: build-server start-server stop-server wait-for-server + +PID_FILE := server.pid +PORT := 8097 + +build/s3ec-server: + mkdir -p build && cd build && cmake .. + +build-server: | build/s3ec-server + @echo "Building Cpp transition server..." + cd build && $(MAKE) + +start-server: + @echo "Starting Cpp transition server..." + cd build && \ + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + ./s3ec-server > ../server.log 2>&1 & echo $$! > ../$(PID_FILE) + @echo "Cpp transition server starting..." + +stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true + @if [ -f $(PID_FILE) ]; then \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ + fi + @rm -f server.log + @echo "Server stopped" + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT) diff --git a/test-server/cpp-v2-transition-server/README.md b/test-server/cpp-v2-transition-server/README.md new file mode 100644 index 00000000..8e77feda --- /dev/null +++ b/test-server/cpp-v2-transition-server/README.md @@ -0,0 +1,37 @@ +# C++ S3 Encryption Test Server + +Minimal C++ implementation of the S3 Encryption test server. + +## Dependencies + +- libmicrohttpd +- AWS SDK for C++ +- nlohmann/json +- uuid + +On MacOS you can +```bash +brew install libmicrohttpd nlohmann-json ossp-uuid +``` + +## Build + +```bash +mkdir build && cd build +cmake .. +make +``` + +## Run + +```bash +./s3ec-server +``` + +Server runs on localhost:8085 + +## API Endpoints + +- `POST /client` - Create S3 encryption client +- `GET /object/{bucket}/{key}` - Get encrypted object +- `PUT /object/{bucket}/{key}` - Put encrypted object \ No newline at end of file diff --git a/test-server/cpp-v2-transition-server/aws-sdk-cpp b/test-server/cpp-v2-transition-server/aws-sdk-cpp new file mode 160000 index 00000000..9110b0ff --- /dev/null +++ b/test-server/cpp-v2-transition-server/aws-sdk-cpp @@ -0,0 +1 @@ +Subproject commit 9110b0ff85094134a4f78316485fd7fe754a2a9c diff --git a/test-server/cpp-v2-transition-server/main.cpp b/test-server/cpp-v2-transition-server/main.cpp new file mode 100644 index 00000000..9e9f942d --- /dev/null +++ b/test-server/cpp-v2-transition-server/main.cpp @@ -0,0 +1,748 @@ +/* + * S3 Encryption Test Server - C++ V2 Transition + * + * CONCURRENCY AND SYNCHRONIZATION DESIGN: + * + * 1. Threading Model: + * - Uses MHD_USE_POLL_INTERNALLY with fixed thread pool + * - Thread pool size = CPU cores * 2 (auto-detected at startup) + * - Threads are reused across connections for efficiency + * - I/O multiplexing (poll) distributes connections across thread pool + * - All S3 operations are SYNCHRONOUS - server waits for S3 completion before responding + * - POLL mechanism avoids FD_SETSIZE=1024 limitation of select() + * + * 2. Resource Scaling: + * - All limits automatically scale with detected CPU count: + * * Thread pool size = num_cores * 2 + * * Connection limit = num_cores * 2 + * * S3 client maxConnections = num_cores * 2 + * - Multiplier of 2 accounts for I/O blocking without starving throughput + * - Ensures optimal resource usage on any hardware configuration + * + * 3. Client Cache (client_cache_secret): + * - Protected by std::shared_mutex for efficient concurrent access + * - get_client() uses shared_lock (multiple threads can read simultaneously) + * - set_client() uses unique_lock (exclusive write access) + * - This allows concurrent GET/PUT operations without serialization + * - UUID-based keys guarantee uniqueness (always insert, never update) + * + * 4. Memory Management: + * - Request body allocated in request_handler (*con_cls = new std::string()) + * - Body lifetime managed by libmicrohttpd - valid until request_completed() + * - All handler functions complete synchronously before returning + * - request_completed() safely deletes body after response sent + * - No memory leaks under sustained concurrent load + * + * 5. Synchronous Operation Guarantees: + * - GetObject: Waits for S3, reads full response stream, then returns + * - PutObject: Waits for S3 operation to complete, then returns + * - No async callbacks or background operations + * - Client receives response only after S3 operation completes + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using json = nlohmann::json; +using namespace Aws::S3Encryption; +using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; + +// LRU cache for S3 encryption clients +// Limits memory and connection pool growth by evicting least recently used clients +const size_t MAX_CACHED_CLIENTS = 100; // Reasonable limit for concurrent test operations + +struct ClientCacheEntry { + std::shared_ptr client; + std::list::iterator lru_iter; +}; + +std::unordered_map client_cache_secret; +std::list lru_order; // Most recently used at front +std::shared_timed_mutex client_mutex; // Using shared_timed_mutex (C++14 compatible) for concurrent reads + +// Threading configuration - set at startup based on CPU cores +unsigned int g_thread_pool_size = 8; // Default, will be overwritten in main() + +std::string generate_uuid() { + uuid_t uuid; + uuid_generate(uuid); + char uuid_str[37]; + uuid_unparse(uuid, uuid_str); + return std::string(uuid_str); +} + +std::shared_ptr get_client(const std::string &client_id) +{ + // Need unique_lock to update LRU order even on reads + std::unique_lock lock(client_mutex); + auto it = client_cache_secret.find(client_id); + if (it == client_cache_secret.end()) { + return std::shared_ptr(); + } else { + // Move to front of LRU list (mark as most recently used) + lru_order.erase(it->second.lru_iter); + lru_order.push_front(client_id); + it->second.lru_iter = lru_order.begin(); + + return it->second.client; + } +} + +void set_client(const std::string &client_id, std::shared_ptr client) +{ + // UUID guarantees unique keys - always insert, never update + // Still need exclusive lock because std::unordered_map isn't thread-safe for concurrent inserts + std::unique_lock lock(client_mutex); + + // Add to front of LRU list (most recently used) + lru_order.push_front(client_id); + + ClientCacheEntry entry; + entry.client = client; + entry.lru_iter = lru_order.begin(); + + client_cache_secret.emplace(client_id, entry); + + // Evict least recently used clients if we exceed the limit + while (client_cache_secret.size() > MAX_CACHED_CLIENTS) { + std::string lru_client_id = lru_order.back(); + lru_order.pop_back(); + + auto evict_it = client_cache_secret.find(lru_client_id); + if (evict_it != client_cache_secret.end()) { + fprintf(stderr, "[CPP-V2-TRANSITION] [CACHE-EVICT] Evicting client %s (cache size was %zu)\n", + lru_client_id.c_str(), client_cache_secret.size()); + client_cache_secret.erase(evict_it); + } + } + + fprintf(stderr, "[CPP-V2-TRANSITION] [CACHE-ADD] Added client %s (cache size now %zu)\n", + client_id.c_str(), client_cache_secret.size()); +} + +std::string get_header_value(struct MHD_Connection *connection, + const char *key) { + const char *value = + MHD_lookup_connection_value(connection, MHD_HEADER_KIND, key); + return value ? std::string(value) : ""; +} + +MHD_Result send_response(struct MHD_Connection *connection, int status_code, + const std::string &content) { + struct MHD_Response *response = MHD_create_response_from_buffer( + content.length(), (void *)content.data(), MHD_RESPMEM_MUST_COPY); + MHD_Result ret = MHD_queue_response(connection, status_code, response); + MHD_destroy_response(response); + return ret; +} + +std::string make_error(const std::string &message, int status_code) { + return "{\"__type\": " + "\"software.amazon.encryption.s3#S3EncryptionClientError\", " + "\"message\": \"" + + message + "\"}"; +} + +bool unsupported(std::string& commitmentPolicy, std::string& encryptionAlgorithm) +{ + if (encryptionAlgorithm == "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY") return true; + if (commitmentPolicy == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT") return true; + if (commitmentPolicy == "REQUIRE_ENCRYPT_ALLOW_DECRYPT") return true; + return false; +} + +std::string get_config(json & request, const char * x) +{ + if (!request.contains("config")) return ""; + auto config = request["config"]; + if (config.contains(x)) + return config[x]; + return ""; +} + +MHD_Result handle_create_client(struct MHD_Connection *connection, + const std::string &body) { + // Body is kept alive by *con_cls until request_completed fires, so it's safe to use directly + // All operations here are synchronous and complete before returning to caller + + try { + json request = json::parse(body); + std::string commitmentPolicy = get_config(request, "commitmentPolicy"); + std::string encryptionAlgorithm = get_config(request, "encryptionAlgorithm"); + + if (unsupported(commitmentPolicy, encryptionAlgorithm)) { + send_response(connection, 404, "{\"error\":\"Unsupported Option.\"}"); + return MHD_YES; + } + + // Extract all key material types + std::string kms_key_id; + std::string rsa_key_blob; + std::string aes_key_blob; + + if (request["config"]["keyMaterial"].contains("kmsKeyId") && + !request["config"]["keyMaterial"]["kmsKeyId"].is_null()) { + kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; + } + if (request["config"]["keyMaterial"].contains("rsaKey") && + !request["config"]["keyMaterial"]["rsaKey"].is_null()) { + rsa_key_blob = request["config"]["keyMaterial"]["rsaKey"]; + } + if (request["config"]["keyMaterial"].contains("aesKey") && + !request["config"]["keyMaterial"]["aesKey"].is_null()) { + aes_key_blob = request["config"]["keyMaterial"]["aesKey"]; + } + + // Validate that only one key type is provided + int key_count = 0; + if (!kms_key_id.empty()) key_count++; + if (!rsa_key_blob.empty()) key_count++; + if (!aes_key_blob.empty()) key_count++; + + if (key_count != 1) { + return send_response(connection, 400, + "{\"error\":\"KeyMaterial must contain exactly one non-null key type\"}"); + } + + // RSA is not supported by C++ SDK + if (!rsa_key_blob.empty()) { + return send_response(connection, 501, + "{\"error\":\"RSA key wrapping is not supported in C++ S3 Encryption Client\"}"); + } + + bool legacy1 = request["config"]["enableLegacyWrappingAlgorithms"]; + bool legacy2 = request["config"]["enableLegacyUnauthenticatedModes"]; + bool inst_put = false; + if (request["config"].contains("instructionFileConfig") && + request["config"]["instructionFileConfig"].contains("enableInstructionFilePutObject")) { + inst_put = request["config"]["instructionFileConfig"]["enableInstructionFilePutObject"]; + } + + // Create CryptoConfigurationV2 based on key type + std::shared_ptr config; + + if (!aes_key_blob.empty()) { + // Base64 decode the AES key + Aws::Utils::ByteBuffer decoded = Aws::Utils::HashingUtils::Base64Decode(aes_key_blob); + if (decoded.GetLength() == 0) { + return send_response(connection, 400, + "{\"error\":\"Failed to decode AES key\"}"); + } + + Aws::Utils::CryptoBuffer key_buffer( + decoded.GetUnderlyingData(), + decoded.GetLength() + ); + + auto materials = std::make_shared< + Aws::S3Encryption::Materials::SimpleEncryptionMaterialsWithGCMAAD>( + key_buffer + ); + config = std::make_shared(materials); + } else if (!kms_key_id.empty()) { + auto materials = std::make_shared(kms_key_id); + config = std::make_shared(materials); + } else { + return send_response(connection, 400, + "{\"error\":\"No valid key material provided\"}"); + } + + // Apply common configuration settings (applies to both AES and KMS) + if (legacy1 || legacy2) + config->SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); + if (legacy2) + config->SetUnAuthenticatedRangeGet(RangeGetMode::ALL); + if (inst_put) + config->SetStorageMethod(StorageMethod::INSTRUCTION_FILE); + + // Create S3EncryptionClientV2 with standard configuration + Aws::Client::ClientConfiguration clientConfig; + clientConfig.maxConnections = 512; // Large pool per client + clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); + auto encryption_client = std::make_shared(*config, clientConfig); + + std::string client_id = generate_uuid(); + set_client(client_id, encryption_client); + + json response = {{"clientId", client_id}}; + return send_response(connection, 200, response.dump()); + } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V2-TRANSITION] handle_create_client exception: %s\n", e.what()); + return send_response(connection, 500, + "{\"error\":\"An exception was thrown.\"}"); + } catch (...) { + return send_response(connection, 500, "{\"error\":\"Unknown error\"}"); + } +} + +void fill_context(Aws::Map &map, + const std::string &metadata) { + if (metadata.empty()) { + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] fill_context: metadata is empty\n"); + return; + } + + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] fill_context: raw metadata='%s' (length=%zu)\n", + metadata.c_str(), metadata.length()); + + // Parse metadata format: [key1]:[value1],[key2]:[value2],... + // or single pair: [key]:[value] + std::string current = metadata; + size_t pos = 0; + int pair_count = 0; + + while (pos < current.length()) { + // Find opening bracket for key + size_t key_start = current.find('[', pos); + if (key_start == std::string::npos) + break; + + // Find closing bracket for key + size_t key_end = current.find(']', key_start); + if (key_end == std::string::npos) + break; + + // Find colon separator + size_t colon = current.find(':', key_end); + if (colon == std::string::npos) + break; + + // Find opening bracket for value + size_t value_start = current.find('[', colon); + if (value_start == std::string::npos) + break; + + // Find closing bracket for value + size_t value_end = current.find(']', value_start); + if (value_end == std::string::npos) + break; + + // Extract key and value + std::string key = current.substr(key_start + 1, key_end - key_start - 1); + std::string value = + current.substr(value_start + 1, value_end - value_start - 1); + + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] fill_context: parsed pair #%d: key='%s', value='%s'\n", + ++pair_count, key.c_str(), value.c_str()); + + // Add to map + map.emplace(key, value); + + // Move to next pair (look for comma or next opening bracket) + pos = value_end + 1; + size_t comma = current.find(',', pos); + if (comma != std::string::npos) { + pos = comma + 1; + } + } + + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] fill_context: completed, parsed %d pairs into map\n", pair_count); +} + +MHD_Result handle_get_object(struct MHD_Connection *connection, + std::string bucket, + std::string key, + std::string client_id, + std::string metadata, + std::string range) { + // Get thread ID for debugging concurrent operations + std::thread::id thread_id = std::this_thread::get_id(); + + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject START: thread=%lu, bucket=%s, key=%s, client_id=%s, metadata_length=%zu, range=%s\n", + (unsigned long)std::hash{}(thread_id), bucket.c_str(), key.c_str(), client_id.c_str(), metadata.length(), range.c_str()); + + auto client = get_client(client_id); + if (!client) { + fprintf(stderr, "[CPP-V2-TRANSITION] GetObject error: Client not found for client_id=%s\n", client_id.c_str()); + return send_response(connection, 404, "{\"error\":\"Client not found\"}"); + } + + try { + Aws::S3::Model::GetObjectRequest request; + request.SetBucket(bucket); + request.SetKey(key); + + // Add range header if provided + if (!range.empty()) { + request.SetRange(range); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject: Setting range=%s\n", range.c_str()); + } + + Aws::Map kmsContextMap; + fill_context(kmsContextMap, metadata); + + // Log the encryption context map size and contents + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject: encryption context map size=%zu\n", kmsContextMap.size()); + for (const auto& pair : kmsContextMap) { + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject: context['%s']='%s'\n", + pair.first.c_str(), pair.second.c_str()); + } + + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject: calling client->GetObject() for key=%s\n", key.c_str()); + + // Keep outcome alive to ensure stream remains valid + auto outcome = client->GetObject(request, kmsContextMap); + + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject: client->GetObject() returned for key=%s\n", key.c_str()); + + if (outcome.IsSuccess()) { + // Read the stream completely before outcome goes out of scope + auto &stream = outcome.GetResult().GetBody(); + std::stringstream buffer; + buffer << stream.rdbuf(); + std::string content = buffer.str(); + + // Validate we read something + if (content.empty() && stream.fail()) { + fprintf(stderr, "[CPP-V2-TRANSITION] GetObject error: Failed to read stream for bucket=%s, key=%s\n", + bucket.c_str(), key.c_str()); + auto msg = make_error("Failed to read response stream", 500); + return send_response(connection, 500, msg); + } + + fprintf(stderr, "[CPP-V2-TRANSITION] GetObject success: bucket=%s, key=%s, size=%zu bytes\n", + bucket.c_str(), key.c_str(), content.length()); + + // Create and send response + struct MHD_Response *response = MHD_create_response_from_buffer( + content.length(), (void *)content.data(), MHD_RESPMEM_MUST_COPY); + + // Add keep-alive header + MHD_add_response_header(response, "Connection", "keep-alive"); + MHD_add_response_header(response, "Keep-Alive", "timeout=30, max=100"); + + MHD_Result ret = MHD_queue_response(connection, 200, response); + MHD_destroy_response(response); + + return ret; + } else { + // Enhanced error logging with thread info + auto error = outcome.GetError(); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject FAILED: thread=%lu, key=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str()); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject error details:\n"); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] - Message: %s\n", error.GetMessage().c_str()); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] - ExceptionName: %s\n", error.GetExceptionName().c_str()); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] - ResponseCode: %d\n", (int)error.GetResponseCode()); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] - ShouldRetry: %s\n", error.ShouldRetry() ? "true" : "false"); + + auto msg = make_error(outcome.GetError().GetMessage(), 500); + fprintf(stderr, "[CPP-V2-TRANSITION] GetObject AWS error: %s\n", msg.c_str()); + return send_response(connection, 500, msg); + } + } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject EXCEPTION: thread=%lu, key=%s, what=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str(), e.what()); + auto msg = make_error(e.what(), 500); + return send_response(connection, 500, msg); + } catch (...) { + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject UNKNOWN EXCEPTION: thread=%lu, key=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str()); + auto msg = make_error("Unknown error in GetObject", 500); + return send_response(connection, 500, msg); + } +} + +MHD_Result handle_put_object(struct MHD_Connection *connection, + std::string bucket, + std::string key, + std::string client_id, + std::string body, + std::string metadata) { + fprintf(stderr, "[CPP-V2-TRANSITION] PutObject request: bucket=%s, key=%s, client_id=%s, body_size=%zu\n", + bucket.c_str(), key.c_str(), client_id.c_str(), body.length()); + + auto client = get_client(client_id); + if (!client) { + fprintf(stderr, "[CPP-V2-TRANSITION] PutObject error: Client not found for client_id=%s\n", client_id.c_str()); + return send_response(connection, 404, "{\"error\":\"Client not found\"}"); + } + + try { + // Create owned copy of body data to ensure it lives through the S3 operation + auto body_ptr = std::make_shared(body); + + Aws::Map kmsContextMap; + fill_context(kmsContextMap, metadata); + + Aws::S3::Model::PutObjectRequest request; + request.SetBucket(bucket); + request.SetKey(key); + + // Create stream from owned body data + auto stream = std::make_shared(*body_ptr); + request.SetBody(stream); + + // Synchronous call - waits for S3 operation to complete + // body_ptr keeps the data alive through this entire operation + auto outcome = client->PutObject(request, kmsContextMap); + if (outcome.IsSuccess()) { + fprintf(stderr, "[CPP-V2-TRANSITION] PutObject success: bucket=%s, key=%s\n", bucket.c_str(), key.c_str()); + json response = {{"bucket", bucket}, {"key", key}}; + return send_response(connection, 200, response.dump()); + } else { + auto msg = make_error(outcome.GetError().GetMessage(), 500); + fprintf(stderr, "[CPP-V2-TRANSITION] PutObject AWS error: %s\n", msg.c_str()); + return send_response(connection, 500, msg); + } + } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V2-TRANSITION] PutObject exception: %s\n", e.what()); + auto msg = make_error(e.what(), 500); + return send_response(connection, 500, msg); + } +} + +void request_completed(void *cls, struct MHD_Connection *connection, + void **con_cls, enum MHD_RequestTerminationCode toe) { + // Clean up the request-specific context when request is truly complete + // This is called AFTER all handlers have returned and the response has been sent + + // Log why the request was terminated + const char* reason = "UNKNOWN"; + switch (toe) { + case MHD_REQUEST_TERMINATED_COMPLETED_OK: + reason = "COMPLETED_OK"; + break; + case MHD_REQUEST_TERMINATED_WITH_ERROR: + reason = "WITH_ERROR"; + break; + case MHD_REQUEST_TERMINATED_TIMEOUT_REACHED: + reason = "TIMEOUT_REACHED"; + break; + case MHD_REQUEST_TERMINATED_DAEMON_SHUTDOWN: + reason = "DAEMON_SHUTDOWN"; + break; + case MHD_REQUEST_TERMINATED_READ_ERROR: + reason = "READ_ERROR"; + break; + case MHD_REQUEST_TERMINATED_CLIENT_ABORT: + reason = "CLIENT_ABORT"; + break; + } + fprintf(stderr, "[CPP-V2-TRANSITION] request_completed called, reason=%s, con_cls=%p\n", + reason, *con_cls); + + if (*con_cls != nullptr) { + std::string *body = static_cast(*con_cls); + delete body; // Safe to delete now - all synchronous operations are complete + *con_cls = nullptr; + } +} + +MHD_Result request_handler(void *cls, struct MHD_Connection *connection, + const char *url, const char *method, + const char *version, const char *upload_data, + size_t *upload_data_size, void **con_cls) { + try { + std::string method_str(method); + std::string url_str(url); + bool is_push = method_str == "POST" || method_str == "PUT"; + + // LOG: Every request entry (even first-time calls) + if (*con_cls == nullptr) { + fprintf(stderr, "[CPP-V2-TRANSITION] REQUEST START: method=%s, url=%s, version=%s, con_cls=NULL, upload_data_size=%zu\n", + method, url, version, *upload_data_size); + } + + // Initialize request context on first call + if (*con_cls == nullptr) { + // Allocate unique state for each request to avoid race conditions + *con_cls = new std::string(); + fprintf(stderr, "[CPP-V2-TRANSITION] REQUEST INIT: allocated new request context for %s %s\n", method, url); + return MHD_YES; + } + + // LOG: Subsequent calls + if (is_push && *upload_data_size > 0) { + fprintf(stderr, "[CPP-V2-TRANSITION] REQUEST DATA: %s %s receiving %zu bytes\n", method, url, *upload_data_size); + } else if (*upload_data_size == 0) { + fprintf(stderr, "[CPP-V2-TRANSITION] REQUEST COMPLETE: %s %s ready for processing\n", method, url); + } + + // Accumulate request body data for POST/PUT requests + if (is_push && *upload_data_size > 0) { + std::string *body = static_cast(*con_cls); + body->append(upload_data, *upload_data_size); + *upload_data_size = 0; + return MHD_YES; + } + + // At this point, *upload_data_size == 0, meaning we have all the data + // Now we can safely process the request + + // LOG: About to process request + fprintf(stderr, "[CPP-V2-TRANSITION] PROCESSING: %s %s\n", method, url); + + // Handle client creation endpoint + if (is_push && url_str == "/client") { + fprintf(stderr, "[CPP-V2-TRANSITION] Handling /client endpoint\n"); + std::string *body = static_cast(*con_cls); + MHD_Result result = handle_create_client(connection, *body); + fprintf(stderr, "[CPP-V2-TRANSITION] /client handler returned: %d\n", result); + return result; + } + + // Handle object operations + if (url_str.find("/object/") == 0) { + fprintf(stderr, "[CPP-V2-TRANSITION] Handling /object/ endpoint\n"); + std::string path = url_str.substr(8); // Remove "/object/" + size_t slash_pos = path.find('/'); + if (slash_pos != std::string::npos) { + std::string bucket = path.substr(0, slash_pos); + std::string key = path.substr(slash_pos + 1); + std::string client_id = get_header_value(connection, "clientid"); + std::string metadata = get_header_value(connection, "content-metadata"); + + fprintf(stderr, "[CPP-V2-TRANSITION] Object operation: bucket=%s, key=%s, client_id=%s, method=%s\n", + bucket.c_str(), key.c_str(), client_id.c_str(), method); + + if (method_str == "GET") { + fprintf(stderr, "[CPP-V2-TRANSITION] Dispatching to handle_get_object\n"); + std::string range = get_header_value(connection, "Range"); + MHD_Result result = handle_get_object(connection, bucket, key, client_id, metadata, range); + fprintf(stderr, "[CPP-V2-TRANSITION] handle_get_object returned: %d\n", result); + return result; + } else if (method_str == "PUT") { + fprintf(stderr, "[CPP-V2-TRANSITION] Dispatching to handle_put_object\n"); + std::string *body = static_cast(*con_cls); + MHD_Result result = handle_put_object(connection, bucket, key, client_id, *body, metadata); + fprintf(stderr, "[CPP-V2-TRANSITION] handle_put_object returned: %d\n", result); + return result; + } else { + fprintf(stderr, "[CPP-V2-TRANSITION] Method not allowed: %s\n", method); + return send_response(connection, 405, "{\"error\":\"Method not allowed\"}"); + } + } + } + + // Return error for unrecognized endpoints + fprintf(stderr, "[CPP-V2-TRANSITION] ERROR: Unrecognized endpoint: %s %s\n", method, url); + return send_response(connection, 404, + "{\"error\":\"Not idea what is happening\"}"); + } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V2-TRANSITION] FATAL: Unhandled exception in request_handler: %s (method=%s, url=%s)\n", + e.what(), method, url); + // Try to send error response, but connection might already be broken + try { + return send_response(connection, 500, + "{\"error\":\"Internal server error: unhandled exception\"}"); + } catch (...) { + fprintf(stderr, "[CPP-V2-TRANSITION] FATAL: Failed to send error response\n"); + return MHD_NO; + } + } catch (...) { + fprintf(stderr, "[CPP-V2-TRANSITION] FATAL: Unknown exception in request_handler (method=%s, url=%s)\n", + method, url); + // Try to send error response, but connection might already be broken + try { + return send_response(connection, 500, + "{\"error\":\"Internal server error: unknown exception\"}"); + } catch (...) { + fprintf(stderr, "[CPP-V2-TRANSITION] FATAL: Failed to send error response\n"); + return MHD_NO; + } + } +} + +// Error log callback for libmicrohttpd +void log_mhd_error(void* cls, const char* fmt, va_list ap) { + fprintf(stderr, "[CPP-V2-TRANSITION] [MHD-ERROR] "); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n"); +} + +// Connection notification callback - called when a client connects +MHD_Result notify_connection(void *cls, + struct MHD_Connection *connection, + void **socket_context, + enum MHD_ConnectionNotificationCode toe) { + if (toe == MHD_CONNECTION_NOTIFY_STARTED) { + fprintf(stderr, "[CPP-V2-TRANSITION] [MHD-CONNECT] New connection started\n"); + } else if (toe == MHD_CONNECTION_NOTIFY_CLOSED) { + fprintf(stderr, "[CPP-V2-TRANSITION] [MHD-DISCONNECT] Connection closed\n"); + } + return MHD_YES; +} + +int main() { + Aws::SDKOptions options; + + // Configure AWS SDK logging to output to stderr (which goes to server.log) + // Using Debug level to capture all SDK activity including CryptoModule errors + options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; + options.loggingOptions.logger_create_fn = []() { + return std::make_shared( + Aws::Utils::Logging::LogLevel::Debug + ); + }; + + fprintf(stderr, "[CONFIG] AWS SDK logging enabled at Debug level\n"); + + Aws::InitAPI(options); + + // Detect CPU core count and configure threading + unsigned int num_cores = std::thread::hardware_concurrency(); + if (num_cores == 0) { + num_cores = 4; + fprintf(stderr, "[CPP-V2-TRANSITION] [WARNING] CPU core detection failed, defaulting to %u cores\n", num_cores); + } + + g_thread_pool_size = num_cores * 2; + unsigned int connection_limit = g_thread_pool_size; + + // Log configuration + fprintf(stderr, "[CONFIG] Detected CPU cores: %u\n", num_cores); + fprintf(stderr, "[CONFIG] Thread pool size: %u\n", g_thread_pool_size); + fprintf(stderr, "[CONFIG] Connection limit: %u\n", connection_limit); + fprintf(stderr, "[CONFIG] Each S3 client will use 512 max connections\n"); + + int port = 8097; + + struct MHD_Daemon *daemon = + MHD_start_daemon(MHD_USE_POLL_INTERNALLY | MHD_USE_INTERNAL_POLLING_THREAD | MHD_USE_ERROR_LOG, + port, NULL, NULL, + &request_handler, NULL, + MHD_OPTION_EXTERNAL_LOGGER, log_mhd_error, NULL, + MHD_OPTION_NOTIFY_CONNECTION, notify_connection, NULL, + MHD_OPTION_NOTIFY_COMPLETED, request_completed, NULL, + MHD_OPTION_THREAD_POOL_SIZE, g_thread_pool_size, + MHD_OPTION_CONNECTION_LIMIT, connection_limit, + MHD_OPTION_CONNECTION_TIMEOUT, 10, + MHD_OPTION_END); + + if (!daemon) { + fprintf(stderr, "[CPP-V2-TRANSITION] Failed to start server on port %d\n", port); + Aws::ShutdownAPI(options); + return 1; + } + + fprintf(stderr, "Server running on port %d\n", port); + sleep(10000); + + MHD_stop_daemon(daemon); + Aws::ShutdownAPI(options); + fprintf(stderr, "Ending server\n"); + return 0; +} diff --git a/test-server/cpp-v3-server/.duvet/.gitignore b/test-server/cpp-v3-server/.duvet/.gitignore new file mode 100644 index 00000000..93956e36 --- /dev/null +++ b/test-server/cpp-v3-server/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ \ No newline at end of file diff --git a/test-server/cpp-v3-server/.duvet/config.toml b/test-server/cpp-v3-server/.duvet/config.toml new file mode 100644 index 00000000..3a49ac85 --- /dev/null +++ b/test-server/cpp-v3-server/.duvet/config.toml @@ -0,0 +1,45 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "aws-sdk-cpp/src/aws-cpp-sdk-s3-encryption/**/*.cpp" + +[[source]] +pattern = "aws-sdk-cpp/src/aws-cpp-sdk-s3-encryption/**/*.h" + +[[source]] +pattern = "aws-sdk-cpp/src/aws-cpp-sdk-core/include/aws/core/utils/crypto/*.h" + +[[source]] +pattern = "aws-sdk-cpp/src/aws-cpp-sdk-core/include/aws/core/utils/crypto/*.cpp" + +[[source]] +pattern = "aws-sdk-cpp/tests/aws-cpp-sdk-s3-encryption-tests/*.cpp" + +[[source]] +pattern = "aws-sdk-cpp/tests/aws-cpp-sdk-s3-encryption-integration-tests/*.cpp" + +[[source]] +pattern = "compliance.txt" + +[[specification]] +source = "../specification/s3-encryption/client.md" +[[specification]] +source = "../specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "../specification/s3-encryption/data-format/metadata-strategy.md" +[[specification]] +source = "../specification/s3-encryption/encryption.md" +[[specification]] +source = "../specification/s3-encryption/decryption.md" +[[specification]] +source = "../specification/s3-encryption/key-derivation.md" +[[specification]] +source = "../specification/s3-encryption/key-commitment.md" + + +[report.html] +enabled = true + +# Enable snapshots to prevent requirement coverage regressions +[report.snapshot] +enabled = false diff --git a/test-server/cpp-v3-server/CMakeLists.txt b/test-server/cpp-v3-server/CMakeLists.txt new file mode 100644 index 00000000..0faac5f0 --- /dev/null +++ b/test-server/cpp-v3-server/CMakeLists.txt @@ -0,0 +1,49 @@ +cmake_minimum_required(VERSION 3.16) +project(s3ec-cpp-v2-server) + +set(CMAKE_CXX_STANDARD 17) + +# Configure AWS SDK build options +set(BUILD_ONLY "kms;s3;s3-encryption" CACHE STRING "Build only KMS, S3, and S3-encryption components") +set(ENABLE_TESTING OFF CACHE BOOL "Disable testing") +set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build static libraries") +set(ENABLE_ADDRESS_SANITIZER ON CACHE BOOL "Enable Address Sanitizer") + +# Add AWS SDK as subdirectory +add_subdirectory(aws-sdk-cpp) + +find_package(PkgConfig REQUIRED) +pkg_check_modules(LIBMICROHTTPD REQUIRED libmicrohttpd) + +find_package(nlohmann_json REQUIRED) + +add_executable(s3ec-server main.cpp) + +# Enable Address Sanitizer for the executable +target_compile_options(s3ec-server PRIVATE -fsanitize=address -fno-omit-frame-pointer) +target_link_options(s3ec-server PRIVATE -fsanitize=address) + +target_include_directories(s3ec-server PRIVATE + ${LIBMICROHTTPD_INCLUDE_DIRS} + /opt/homebrew/include +) + +target_include_directories(s3ec-server PRIVATE + ${LIBMICROHTTPD_INCLUDE_DIRS} + /opt/homebrew/include +) + +target_link_directories(s3ec-server PRIVATE + ${LIBMICROHTTPD_LIBRARY_DIRS} + /opt/homebrew/lib +) + +target_link_libraries(s3ec-server + ${LIBMICROHTTPD_LIBRARIES} + aws-cpp-sdk-core + aws-cpp-sdk-kms + aws-cpp-sdk-s3 + aws-cpp-sdk-s3-encryption + nlohmann_json::nlohmann_json + uuid +) diff --git a/test-server/cpp-v3-server/Makefile b/test-server/cpp-v3-server/Makefile new file mode 100644 index 00000000..e90c8d73 --- /dev/null +++ b/test-server/cpp-v3-server/Makefile @@ -0,0 +1,43 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: build-server start-server stop-server wait-for-server + +PID_FILE := server.pid +PORT := 8091 + +build/s3ec-server: + mkdir -p build && cd build && cmake .. + +build-server: | build/s3ec-server + @echo "Building Cpp V3 server..." + cd build && $(MAKE) + +start-server: + @echo "Starting Cpp V3 server..." + cd build && \ + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + ./s3ec-server > ../server.log 2>&1 & echo $$! > ../$(PID_FILE) + @echo "Cpp V3 server starting..." + +stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true + @if [ -f $(PID_FILE) ]; then \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ + fi + @rm -f server.log + @echo "Server stopped" + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT) + +duvet: + duvet report + +view-report-mac: + open .duvet/reports/report.html diff --git a/test-server/cpp-v3-server/README.md b/test-server/cpp-v3-server/README.md new file mode 100644 index 00000000..8e77feda --- /dev/null +++ b/test-server/cpp-v3-server/README.md @@ -0,0 +1,37 @@ +# C++ S3 Encryption Test Server + +Minimal C++ implementation of the S3 Encryption test server. + +## Dependencies + +- libmicrohttpd +- AWS SDK for C++ +- nlohmann/json +- uuid + +On MacOS you can +```bash +brew install libmicrohttpd nlohmann-json ossp-uuid +``` + +## Build + +```bash +mkdir build && cd build +cmake .. +make +``` + +## Run + +```bash +./s3ec-server +``` + +Server runs on localhost:8085 + +## API Endpoints + +- `POST /client` - Create S3 encryption client +- `GET /object/{bucket}/{key}` - Get encrypted object +- `PUT /object/{bucket}/{key}` - Put encrypted object \ No newline at end of file diff --git a/test-server/cpp-v3-server/aws-sdk-cpp b/test-server/cpp-v3-server/aws-sdk-cpp new file mode 160000 index 00000000..9110b0ff --- /dev/null +++ b/test-server/cpp-v3-server/aws-sdk-cpp @@ -0,0 +1 @@ +Subproject commit 9110b0ff85094134a4f78316485fd7fe754a2a9c diff --git a/test-server/cpp-v3-server/compliance.txt b/test-server/cpp-v3-server/compliance.txt new file mode 100644 index 00000000..8225d8a9 --- /dev/null +++ b/test-server/cpp-v3-server/compliance.txt @@ -0,0 +1,119 @@ +** The C++ S3EC does not support re-encryption, nor custom instruction file suffixes +//= ../specification/s3-encryption/data-format/metadata-strategy.md#instruction-file +//= type=exception +//# The S3EC MAY support re-encryption/key rotation via Instruction Files. + +//= ../specification/s3-encryption/data-format/metadata-strategy.md#instruction-file +//= type=exception +//# The S3EC SHOULD support providing a custom Instruction File suffix on GetObject requests, regardless of whether or not re-encryption is supported. + +** We're not doing double encoding yet +//= ../specification/s3-encryption/data-format/metadata-strategy.md#object-metadata +//= type=exception +//# The S3EC SHOULD support decoding the S3 Server's "double encoding". + +//= ../specification/s3-encryption/data-format/content-metadata.md#v3-only +//= type=exception +//# This material description string MAY be encoded by the esoteric double-encoding scheme used by the S3 web server. + +//= ../specification/s3-encryption/data-format/content-metadata.md#v3-only +//= type=exception +//# This encryption context string MAY be encoded by the esoteric double-encoding scheme used by the S3 web server. + +//= ../specification/s3-encryption/data-format/content-metadata.md#v1-v2-shared +//= type=exception +//# This string MAY be encoded by the esoteric double-encoding scheme used by the S3 web server. + + +** The C++ S3EC does not support key rings nor cmms +//= ../specification/s3-encryption/client.md#cryptographic-materials +//= type=exception +//# The S3EC MUST accept either one CMM or one Keyring instance upon initialization. +//# If both a CMM and a Keyring are provided, the S3EC MUST throw an exception. +//# When a Keyring is provided, the S3EC MUST create an instance of the DefaultCMM using the provided Keyring. + + +** The C++ S3EC does not support Delayed Authentication buffer size configuration +//= ../specification/s3-encryption/client.md#set-buffer-size +//= type=exception +//# The S3EC SHOULD accept a configurable buffer size which refers to the maximum ciphertext length in bytes to store in memory when Delayed Authentication mode is disabled. +//# If Delayed Authentication mode is enabled, and the buffer size has been set to a value other than its default, the S3EC MUST throw an exception. +//# If Delayed Authentication mode is disabled, and no buffer size is provided, the S3EC MUST set the buffer size to a reasonable default. + + +** In the C++ S3EC, there is no connection between the S3 client and any potential KMS clients +//= ../specification/s3-encryption/client.md#inherited-sdk-configuration +//= type=exception +//# If the S3EC accepts SDK client configuration, the configuration MUST be applied to all wrapped SDK clients including the KMS client. + + +** In the C++ S3EC, the encryption algorithm is uniquely determined by the client version and the CommitmentPolicy + +//= ../specification/s3-encryption/client.md#encryption-algorithm +//= type=exception +//# The S3EC MUST support configuration of the encryption algorithm (or algorithm suite) during its initialization. +//# The S3EC MUST validate that the configured encryption algorithm is not legacy. +//# If the configured encryption algorithm is legacy, then the S3EC MUST throw an exception. + +//= ../specification/s3-encryption/client.md#key-commitment +//= type=exception +//# The S3EC MUST validate the configured Encryption Algorithm against the provided key commitment policy. +//# If the configured Encryption Algorithm is incompatible with the key commitment policy, then it MUST throw an exception. + + +** The C++ S3EC does not accept a source of randomness during client initialization +//= ../specification/s3-encryption/client.md#randomness +//= type=exception +//# The S3EC MAY accept a source of randomness during client initialization. + + +** This is silly, and I don't want to do it +//= ../specification/s3-encryption/encryption.md#cipher-initialization +//= type=exception +//# The client SHOULD validate that the generated IV or Message ID is not zeros. + +** The C++ S3EC does not support custom materials. +** The built in Raw Keyring always has an empty Materials Description +** Therefore "x-amz-m" will never be written. +//= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys +//= type=exception +//# - The mapkey "x-amz-m" SHOULD be present for V3 format objects that use Raw Keyring Material Description. + + +** The C++ S3EC only implements GetObject and PutObject ** + +//= ../specification/s3-encryption/client.md#aws-sdk-compatibility +//= type=exception +//# The S3EC MUST adhere to the same interface for API operations as the conventional AWS SDK S3 client. + +//= ../specification/s3-encryption/client.md#aws-sdk-compatibility +//= type=exception +//# The S3EC SHOULD support invoking operations unrelated to client-side encryption e.g. + +//= ../specification/s3-encryption/client.md#required-api-operations +//= type=exception +//# - DeleteObject MUST be implemented by the S3EC. +//# - DeleteObject MUST delete the given object key. +//# - DeleteObject MUST delete the associated instruction file using the default instruction file suffix. +//# - DeleteObjects MUST be implemented by the S3EC. +//# - DeleteObjects MUST delete each of the given objects. +//# - DeleteObjects MUST delete each of the corresponding instruction files using the default instruction file suffix. + +//= ../specification/s3-encryption/client.md#optional-api-operations +//= type=exception +//# - CreateMultipartUpload MAY be implemented by the S3EC. +//# - If implemented, CreateMultipartUpload MUST initiate a multipart upload. +//# - UploadPart MAY be implemented by the S3EC. +//# - UploadPart MUST encrypt each part. +//# - Each part MUST be encrypted in sequence. +//# - Each part MUST be encrypted using the same cipher instance for each part. +//# - CompleteMultipartUpload MAY be implemented by the S3EC. +//# - CompleteMultipartUpload MUST complete the multipart upload. +//# - AbortMultipartUpload MAY be implemented by the S3EC. +//# - AbortMultipartUpload MUST abort the multipart upload. + +//= ../specification/s3-encryption/client.md#optional-api-operations +//= type=exception +//# - ReEncryptInstructionFile MAY be implemented by the S3EC. +//# - ReEncryptInstructionFile MUST decrypt the instruction file's encrypted data key for the given object using the client's CMM. +//# - ReEncryptInstructionFile MUST re-encrypt the plaintext data key with a provided keyring. diff --git a/test-server/cpp-v3-server/main.cpp b/test-server/cpp-v3-server/main.cpp new file mode 100644 index 00000000..169fa517 --- /dev/null +++ b/test-server/cpp-v3-server/main.cpp @@ -0,0 +1,776 @@ +/* + * S3 Encryption Test Server - C++ V3 + * + * CONCURRENCY AND SYNCHRONIZATION DESIGN: + * + * 1. Threading Model: + * - Uses MHD_USE_POLL_INTERNALLY with fixed thread pool + * - Thread pool size = CPU cores * 2 (auto-detected at startup) + * - Threads are reused across connections for efficiency + * - I/O multiplexing (poll) distributes connections across thread pool + * - All S3 operations are SYNCHRONOUS - server waits for S3 completion before responding + * - POLL mechanism avoids FD_SETSIZE=1024 limitation of select() + * + * 2. Resource Scaling: + * - All limits automatically scale with detected CPU count: + * * Thread pool size = num_cores * 2 + * * Connection limit = num_cores * 2 + * * S3 client maxConnections = num_cores * 2 + * - Multiplier of 2 accounts for I/O blocking without starving throughput + * - Ensures optimal resource usage on any hardware configuration + * + * 3. Client Cache (client_cache_secret): + * - Protected by std::shared_mutex for efficient concurrent access + * - get_client() uses shared_lock (multiple threads can read simultaneously) + * - set_client() uses unique_lock (exclusive write access) + * - This allows concurrent GET/PUT operations without serialization + * - UUID-based keys guarantee uniqueness (always insert, never update) + * + * 4. Memory Management: + * - Request body allocated in request_handler (*con_cls = new std::string()) + * - Body lifetime managed by libmicrohttpd - valid until request_completed() + * - All handler functions complete synchronously before returning + * - request_completed() safely deletes body after response sent + * - No memory leaks under sustained concurrent load + * + * 5. Synchronous Operation Guarantees: + * - GetObject: Waits for S3, reads full response stream, then returns + * - PutObject: Waits for S3 operation to complete, then returns + * - No async callbacks or background operations + * - Client receives response only after S3 operation completes + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using json = nlohmann::json; +using namespace Aws::S3Encryption; +using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; + +// LRU cache for S3 encryption clients +// Limits memory and connection pool growth by evicting least recently used clients +const size_t MAX_CACHED_CLIENTS = 100; // Reasonable limit for concurrent test operations + +struct ClientCacheEntry { + std::shared_ptr client; + std::list::iterator lru_iter; +}; + +std::unordered_map client_cache_secret; +std::list lru_order; // Most recently used at front +std::shared_timed_mutex client_mutex; // Using shared_timed_mutex (C++14 compatible) for concurrent reads + +// Threading configuration - set at startup based on CPU cores +unsigned int g_thread_pool_size = 8; // Default, will be overwritten in main() + +std::string generate_uuid() { + uuid_t uuid; + uuid_generate(uuid); + char uuid_str[37]; + uuid_unparse(uuid, uuid_str); + return std::string(uuid_str); +} + +std::shared_ptr get_client(const std::string &client_id) +{ + // Need unique_lock to update LRU order even on reads + std::unique_lock lock(client_mutex); + auto it = client_cache_secret.find(client_id); + if (it == client_cache_secret.end()) { + return std::shared_ptr(); + } else { + // Move to front of LRU list (mark as most recently used) + lru_order.erase(it->second.lru_iter); + lru_order.push_front(client_id); + it->second.lru_iter = lru_order.begin(); + + return it->second.client; + } +} + +void set_client(const std::string &client_id, std::shared_ptr client) +{ + // UUID guarantees unique keys - always insert, never update + // Still need exclusive lock because std::unordered_map isn't thread-safe for concurrent inserts + std::unique_lock lock(client_mutex); + + // Add to front of LRU list (most recently used) + lru_order.push_front(client_id); + + ClientCacheEntry entry; + entry.client = client; + entry.lru_iter = lru_order.begin(); + + client_cache_secret.emplace(client_id, entry); + + // Evict least recently used clients if we exceed the limit + while (client_cache_secret.size() > MAX_CACHED_CLIENTS) { + std::string lru_client_id = lru_order.back(); + lru_order.pop_back(); + + auto evict_it = client_cache_secret.find(lru_client_id); + if (evict_it != client_cache_secret.end()) { + fprintf(stderr, "[CPP-V3] [CACHE-EVICT] Evicting client %s (cache size was %zu)\n", + lru_client_id.c_str(), client_cache_secret.size()); + client_cache_secret.erase(evict_it); + } + } + + fprintf(stderr, "[CPP-V3] [CACHE-ADD] Added client %s (cache size now %zu)\n", + client_id.c_str(), client_cache_secret.size()); +} + +std::string get_header_value(struct MHD_Connection *connection, + const char *key) { + const char *value = + MHD_lookup_connection_value(connection, MHD_HEADER_KIND, key); + return value ? std::string(value) : ""; +} + +MHD_Result send_response(struct MHD_Connection *connection, int status_code, + const std::string &content) { + struct MHD_Response *response = MHD_create_response_from_buffer( + content.length(), (void *)content.data(), MHD_RESPMEM_MUST_COPY); + MHD_Result ret = MHD_queue_response(connection, status_code, response); + MHD_destroy_response(response); + return ret; +} + +std::string make_error(const std::string &message, int status_code) { + return "{\"__type\": " + "\"software.amazon.encryption.s3#S3EncryptionClientError\", " + "\"message\": \"" + + message + "\"}"; +} + +MHD_Result unsupported(struct MHD_Connection *connection, std::string & commitmentPolicy, std::string & encryptionAlgorithm) { + fprintf(stderr, "Unsupported %s %s\n",commitmentPolicy.c_str(), encryptionAlgorithm.c_str() ); + send_response(connection, 404, "{\"error\":\"Unsupported Option.\"}"); + return MHD_YES; +} + +std::string get_config(json & request, const char * x) +{ + if (!request.contains("config")) return ""; + auto config = request["config"]; + if (config.contains(x)) + return config[x]; + return ""; +} + +MHD_Result handle_create_client(struct MHD_Connection *connection, + const std::string &body) { + // Body is kept alive by *con_cls until request_completed fires, so it's safe to use directly + // All operations here are synchronous and complete before returning to caller + + try { + json request = json::parse(body); + + // Extract all key material types + std::string kms_key_id; + std::string rsa_key_blob; + std::string aes_key_blob; + + if (request["config"]["keyMaterial"].contains("kmsKeyId") && + !request["config"]["keyMaterial"]["kmsKeyId"].is_null()) { + kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; + } + if (request["config"]["keyMaterial"].contains("rsaKey") && + !request["config"]["keyMaterial"]["rsaKey"].is_null()) { + rsa_key_blob = request["config"]["keyMaterial"]["rsaKey"]; + } + if (request["config"]["keyMaterial"].contains("aesKey") && + !request["config"]["keyMaterial"]["aesKey"].is_null()) { + aes_key_blob = request["config"]["keyMaterial"]["aesKey"]; + } + + // Validate that only one key type is provided + int key_count = 0; + if (!kms_key_id.empty()) key_count++; + if (!rsa_key_blob.empty()) key_count++; + if (!aes_key_blob.empty()) key_count++; + + if (key_count != 1) { + return send_response(connection, 400, + "{\"error\":\"KeyMaterial must contain exactly one non-null key type\"}"); + } + + // RSA is not supported by C++ SDK + if (!rsa_key_blob.empty()) { + return send_response(connection, 501, + "{\"error\":\"RSA key wrapping is not supported in C++ S3 Encryption Client\"}"); + } + + bool legacy1 = request["config"]["enableLegacyWrappingAlgorithms"]; + bool legacy2 = request["config"]["enableLegacyUnauthenticatedModes"]; + bool inst_put = false; + if (request["config"].contains("instructionFileConfig") && + request["config"]["instructionFileConfig"].contains("enableInstructionFilePutObject")) { + inst_put = request["config"]["instructionFileConfig"]["enableInstructionFilePutObject"]; + } + + std::string commitmentPolicy = get_config(request, "commitmentPolicy"); + std::string encryptionAlgorithm = get_config(request, "encryptionAlgorithm"); + + // Create CryptoConfigurationV3 based on key type + std::optional config; + + if (!aes_key_blob.empty()) { + // Base64 decode the AES key + Aws::Utils::ByteBuffer decoded = Aws::Utils::HashingUtils::Base64Decode(aes_key_blob); + if (decoded.GetLength() == 0) { + return send_response(connection, 400, + "{\"error\":\"Failed to decode AES key\"}"); + } + + Aws::Utils::CryptoBuffer key_buffer( + decoded.GetUnderlyingData(), + decoded.GetLength() + ); + + auto materials = std::make_shared< + Aws::S3Encryption::Materials::SimpleEncryptionMaterialsWithGCMAAD>( + key_buffer + ); + config.emplace(materials); + } else if (!kms_key_id.empty()) { + auto materials = std::make_shared(kms_key_id); + config.emplace(materials); + } else { + return send_response(connection, 400, + "{\"error\":\"No valid key material provided\"}"); + } + + // Apply common configuration settings (applies to both AES and KMS) + if (legacy1 || legacy2) + config->AllowLegacy(); + if (legacy2) + config->SetUnAuthenticatedRangeGet(RangeGetMode::ALL); + if (inst_put) + config->SetStorageMethod(StorageMethod::INSTRUCTION_FILE); + + // Configure commitment policy (applies to both AES and KMS) + if (commitmentPolicy == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT") { + if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF" || + encryptionAlgorithm == "ALG_AES_256_CBC_IV16_NO_KDF") { + return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + } + config->SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT); + } else if (commitmentPolicy == "REQUIRE_ENCRYPT_ALLOW_DECRYPT") { + if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") { + return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + } + config->SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_ALLOW_DECRYPT); + } else if (commitmentPolicy == "FORBID_ENCRYPT_ALLOW_DECRYPT") { + if (encryptionAlgorithm == "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY") { + return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + } + config->SetCommitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT); + } + + // Create S3EncryptionClientV3 with standard configuration + Aws::Client::ClientConfiguration clientConfig; + clientConfig.maxConnections = 512; // Large pool per client + clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); + + // Increase timeouts for CI environments where SSL handshakes can be slow + // Default connectTimeoutMs is 1000ms, which is too short for busy CI runners + clientConfig.connectTimeoutMs = 10000; // 10 seconds for SSL connection establishment + clientConfig.requestTimeoutMs = 30000; // 30 seconds for complete request/response + + // Disable automatic checksum calculation for encrypted streams + // The ChecksumInterceptor cannot handle non-seekable SymmetricCryptoStream + // which causes intermittent "BadDigest: CRC64NVME you specified did not match" errors + // when the stream gets consumed during checksum calculation and can't be rewound + clientConfig.checksumConfig.requestChecksumCalculation = + Aws::Client::RequestChecksumCalculation::WHEN_REQUIRED; + + auto encryption_client = std::make_shared(*config, clientConfig); + + std::string client_id = generate_uuid(); + set_client(client_id, encryption_client); + + json response = {{"clientId", client_id}}; + return send_response(connection, 200, response.dump()); + } catch (const std::exception &e) { + fprintf(stderr, "handle_create_client exception %s\n", e.what()); + return send_response(connection, 500, + "{\"error\":\"An exception was thrown.\"}"); + } catch (...) { + return send_response(connection, 500, "{\"error\":\"Unknown error\"}"); + } +} + +void fill_context(Aws::Map &map, + const std::string &metadata) { + if (metadata.empty()) { + fprintf(stderr, "[CPP-V3] [DEBUG] fill_context: metadata is empty\n"); + return; + } + + fprintf(stderr, "[CPP-V3] [DEBUG] fill_context: raw metadata='%s' (length=%zu)\n", + metadata.c_str(), metadata.length()); + + // Parse metadata format: [key1]:[value1],[key2]:[value2],... + // or single pair: [key]:[value] + std::string current = metadata; + size_t pos = 0; + int pair_count = 0; + + while (pos < current.length()) { + // Find opening bracket for key + size_t key_start = current.find('[', pos); + if (key_start == std::string::npos) + break; + + // Find closing bracket for key + size_t key_end = current.find(']', key_start); + if (key_end == std::string::npos) + break; + + // Find colon separator + size_t colon = current.find(':', key_end); + if (colon == std::string::npos) + break; + + // Find opening bracket for value + size_t value_start = current.find('[', colon); + if (value_start == std::string::npos) + break; + + // Find closing bracket for value + size_t value_end = current.find(']', value_start); + if (value_end == std::string::npos) + break; + + // Extract key and value + std::string key = current.substr(key_start + 1, key_end - key_start - 1); + std::string value = + current.substr(value_start + 1, value_end - value_start - 1); + + fprintf(stderr, "[CPP-V3] [DEBUG] fill_context: parsed pair #%d: key='%s', value='%s'\n", + ++pair_count, key.c_str(), value.c_str()); + + // Add to map + map.emplace(key, value); + + // Move to next pair (look for comma or next opening bracket) + pos = value_end + 1; + size_t comma = current.find(',', pos); + if (comma != std::string::npos) { + pos = comma + 1; + } + } + + fprintf(stderr, "[CPP-V3] [DEBUG] fill_context: completed, parsed %d pairs into map\n", pair_count); +} + +MHD_Result handle_get_object(struct MHD_Connection *connection, + std::string bucket, + std::string key, + std::string client_id, + std::string metadata, + std::string range) { + // Get thread ID for debugging concurrent operations + std::thread::id thread_id = std::this_thread::get_id(); + + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject START: thread=%lu, bucket=%s, key=%s, client_id=%s, metadata_length=%zu, range=%s\n", + (unsigned long)std::hash{}(thread_id), bucket.c_str(), key.c_str(), client_id.c_str(), metadata.length(), range.c_str()); + + auto client = get_client(client_id); + if (!client) { + fprintf(stderr, "[CPP-V3] GetObject error: Client not found for client_id=%s\n", client_id.c_str()); + return send_response(connection, 404, "{\"error\":\"Client not found\"}"); + } + + try { + Aws::S3::Model::GetObjectRequest request; + request.SetBucket(bucket); + request.SetKey(key); + + // Add range header if provided + if (!range.empty()) { + request.SetRange(range); + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject: Setting range=%s\n", range.c_str()); + } + + Aws::Map kmsContextMap; + fill_context(kmsContextMap, metadata); + + // Log the encryption context map size and contents + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject: encryption context map size=%zu\n", kmsContextMap.size()); + for (const auto& pair : kmsContextMap) { + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject: context['%s']='%s'\n", + pair.first.c_str(), pair.second.c_str()); + } + + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject: calling client->GetObject() for key=%s\n", key.c_str()); + + // Keep outcome alive to ensure stream remains valid + auto outcome = client->GetObject(request, kmsContextMap); + + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject: client->GetObject() returned for key=%s\n", key.c_str()); + + if (outcome.IsSuccess()) { + // Read the stream completely before outcome goes out of scope + auto &stream = outcome.GetResult().GetBody(); + std::stringstream buffer; + buffer << stream.rdbuf(); + std::string content = buffer.str(); + + // Validate we read something + if (content.empty() && stream.fail()) { + fprintf(stderr, "[CPP-V3] GetObject error: Failed to read stream for bucket=%s, key=%s\n", + bucket.c_str(), key.c_str()); + auto msg = make_error("Failed to read response stream", 500); + return send_response(connection, 500, msg); + } + + fprintf(stderr, "[CPP-V3] GetObject success: bucket=%s, key=%s, size=%zu bytes\n", + bucket.c_str(), key.c_str(), content.length()); + + // Create and send response + struct MHD_Response *response = MHD_create_response_from_buffer( + content.length(), (void *)content.data(), MHD_RESPMEM_MUST_COPY); + + // Add keep-alive header + MHD_add_response_header(response, "Connection", "keep-alive"); + MHD_add_response_header(response, "Keep-Alive", "timeout=30, max=100"); + + MHD_Result ret = MHD_queue_response(connection, 200, response); + MHD_destroy_response(response); + + return ret; + } else { + // Enhanced error logging with thread info + auto error = outcome.GetError(); + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject FAILED: thread=%lu, key=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str()); + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject error details:\n"); + fprintf(stderr, "[CPP-V3] [DEBUG] - Message: %s\n", error.GetMessage().c_str()); + fprintf(stderr, "[CPP-V3] [DEBUG] - ExceptionName: %s\n", error.GetExceptionName().c_str()); + fprintf(stderr, "[CPP-V3] [DEBUG] - ResponseCode: %d\n", (int)error.GetResponseCode()); + fprintf(stderr, "[CPP-V3] [DEBUG] - ShouldRetry: %s\n", error.ShouldRetry() ? "true" : "false"); + + auto msg = make_error(outcome.GetError().GetMessage(), 500); + fprintf(stderr, "[CPP-V3] GetObject AWS error: %s\n", msg.c_str()); + return send_response(connection, 500, msg); + } + } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject EXCEPTION: thread=%lu, key=%s, what=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str(), e.what()); + auto msg = make_error(e.what(), 500); + return send_response(connection, 500, msg); + } catch (...) { + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject UNKNOWN EXCEPTION: thread=%lu, key=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str()); + auto msg = make_error("Unknown error in GetObject", 500); + return send_response(connection, 500, msg); + } +} + +MHD_Result handle_put_object(struct MHD_Connection *connection, + std::string bucket, + std::string key, + std::string client_id, + std::string body, + std::string metadata) { + fprintf(stderr, "[CPP-V3] PutObject request: bucket=%s, key=%s, client_id=%s, body_size=%zu\n", + bucket.c_str(), key.c_str(), client_id.c_str(), body.length()); + + auto client = get_client(client_id); + if (!client) { + fprintf(stderr, "[CPP-V3] PutObject error: Client not found for client_id=%s\n", client_id.c_str()); + return send_response(connection, 404, "{\"error\":\"Client not found\"}"); + } + + try { + // Create owned copy of body data to ensure it lives through the S3 operation + auto body_ptr = std::make_shared(body); + + Aws::Map kmsContextMap; + fill_context(kmsContextMap, metadata); + + Aws::S3::Model::PutObjectRequest request; + request.SetBucket(bucket); + request.SetKey(key); + + // Create stream from owned body data + auto stream = std::make_shared(*body_ptr); + request.SetBody(stream); + + // Synchronous call - waits for S3 operation to complete + // body_ptr keeps the data alive through this entire operation + auto outcome = client->PutObject(request, kmsContextMap); + if (outcome.IsSuccess()) { + fprintf(stderr, "[CPP-V3] PutObject success: bucket=%s, key=%s\n", bucket.c_str(), key.c_str()); + json response = {{"bucket", bucket}, {"key", key}}; + return send_response(connection, 200, response.dump()); + } else { + auto msg = make_error(outcome.GetError().GetMessage(), 500); + fprintf(stderr, "[CPP-V3] PutObject AWS error: %s\n", msg.c_str()); + return send_response(connection, 500, msg); + } + } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V3] PutObject exception: %s\n", e.what()); + auto msg = make_error(e.what(), 500); + return send_response(connection, 500, msg); + } +} + +void request_completed(void *cls, struct MHD_Connection *connection, + void **con_cls, enum MHD_RequestTerminationCode toe) { + // Clean up the request-specific context when request is truly complete + // This is called AFTER all handlers have returned and the response has been sent + + // Log why the request was terminated + const char* reason = "UNKNOWN"; + switch (toe) { + case MHD_REQUEST_TERMINATED_COMPLETED_OK: + reason = "COMPLETED_OK"; + break; + case MHD_REQUEST_TERMINATED_WITH_ERROR: + reason = "WITH_ERROR"; + break; + case MHD_REQUEST_TERMINATED_TIMEOUT_REACHED: + reason = "TIMEOUT_REACHED"; + break; + case MHD_REQUEST_TERMINATED_DAEMON_SHUTDOWN: + reason = "DAEMON_SHUTDOWN"; + break; + case MHD_REQUEST_TERMINATED_READ_ERROR: + reason = "READ_ERROR"; + break; + case MHD_REQUEST_TERMINATED_CLIENT_ABORT: + reason = "CLIENT_ABORT"; + break; + } + fprintf(stderr, "[CPP-V3] request_completed called, reason=%s, con_cls=%p\n", + reason, *con_cls); + + if (*con_cls != nullptr) { + std::string *body = static_cast(*con_cls); + delete body; // Safe to delete now - all synchronous operations are complete + *con_cls = nullptr; + } +} + +MHD_Result request_handler(void *cls, struct MHD_Connection *connection, + const char *url, const char *method, + const char *version, const char *upload_data, + size_t *upload_data_size, void **con_cls) { + try { + std::string method_str(method); + std::string url_str(url); + bool is_push = method_str == "POST" || method_str == "PUT"; + + // LOG: Every request entry (even first-time calls) + if (*con_cls == nullptr) { + fprintf(stderr, "[CPP-V3] REQUEST START: method=%s, url=%s, version=%s, con_cls=NULL, upload_data_size=%zu\n", + method, url, version, *upload_data_size); + } + + // Initialize request context on first call + if (*con_cls == nullptr) { + // Allocate unique state for each request to avoid race conditions + *con_cls = new std::string(); + fprintf(stderr, "[CPP-V3] REQUEST INIT: allocated new request context for %s %s\n", method, url); + return MHD_YES; + } + + // LOG: Subsequent calls + if (is_push && *upload_data_size > 0) { + fprintf(stderr, "[CPP-V3] REQUEST DATA: %s %s receiving %zu bytes\n", method, url, *upload_data_size); + } else if (*upload_data_size == 0) { + fprintf(stderr, "[CPP-V3] REQUEST COMPLETE: %s %s ready for processing\n", method, url); + } + + // Accumulate request body data for POST/PUT requests + if (is_push && *upload_data_size > 0) { + std::string *body = static_cast(*con_cls); + body->append(upload_data, *upload_data_size); + *upload_data_size = 0; + return MHD_YES; + } + + // At this point, *upload_data_size == 0, meaning we have all the data + // Now we can safely process the request + + // LOG: About to process request + fprintf(stderr, "[CPP-V3] PROCESSING: %s %s\n", method, url); + + // Handle client creation endpoint + if (is_push && url_str == "/client") { + fprintf(stderr, "[CPP-V3] Handling /client endpoint\n"); + std::string *body = static_cast(*con_cls); + MHD_Result result = handle_create_client(connection, *body); + fprintf(stderr, "[CPP-V3] /client handler returned: %d\n", result); + return result; + } + + // Handle object operations + if (url_str.find("/object/") == 0) { + fprintf(stderr, "[CPP-V3] Handling /object/ endpoint\n"); + std::string path = url_str.substr(8); // Remove "/object/" + size_t slash_pos = path.find('/'); + if (slash_pos != std::string::npos) { + std::string bucket = path.substr(0, slash_pos); + std::string key = path.substr(slash_pos + 1); + std::string client_id = get_header_value(connection, "clientid"); + std::string metadata = get_header_value(connection, "content-metadata"); + + fprintf(stderr, "[CPP-V3] Object operation: bucket=%s, key=%s, client_id=%s, method=%s\n", + bucket.c_str(), key.c_str(), client_id.c_str(), method); + + if (method_str == "GET") { + fprintf(stderr, "[CPP-V3] Dispatching to handle_get_object\n"); + std::string range = get_header_value(connection, "Range"); + MHD_Result result = handle_get_object(connection, bucket, key, client_id, metadata, range); + fprintf(stderr, "[CPP-V3] handle_get_object returned: %d\n", result); + return result; + } else if (method_str == "PUT") { + fprintf(stderr, "[CPP-V3] Dispatching to handle_put_object\n"); + std::string *body = static_cast(*con_cls); + MHD_Result result = handle_put_object(connection, bucket, key, client_id, *body, metadata); + fprintf(stderr, "[CPP-V3] handle_put_object returned: %d\n", result); + return result; + } else { + fprintf(stderr, "[CPP-V3] Method not allowed: %s\n", method); + return send_response(connection, 405, "{\"error\":\"Method not allowed\"}"); + } + } + } + + // Return error for unrecognized endpoints + fprintf(stderr, "[CPP-V3] ERROR: Unrecognized endpoint: %s %s\n", method, url); + return send_response(connection, 404, + "{\"error\":\"Not idea what is happening\"}"); + } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V3] FATAL: Unhandled exception in request_handler: %s (method=%s, url=%s)\n", + e.what(), method, url); + // Try to send error response, but connection might already be broken + try { + return send_response(connection, 500, + "{\"error\":\"Internal server error: unhandled exception\"}"); + } catch (...) { + fprintf(stderr, "[CPP-V3] FATAL: Failed to send error response\n"); + return MHD_NO; + } + } catch (...) { + fprintf(stderr, "[CPP-V3] FATAL: Unknown exception in request_handler (method=%s, url=%s)\n", + method, url); + // Try to send error response, but connection might already be broken + try { + return send_response(connection, 500, + "{\"error\":\"Internal server error: unknown exception\"}"); + } catch (...) { + fprintf(stderr, "[CPP-V3] FATAL: Failed to send error response\n"); + return MHD_NO; + } + } +} + +// Error log callback for libmicrohttpd +void log_mhd_error(void* cls, const char* fmt, va_list ap) { + fprintf(stderr, "[CPP-V3] [MHD-ERROR] "); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n"); +} + +// Connection notification callback - called when a client connects +MHD_Result notify_connection(void *cls, + struct MHD_Connection *connection, + void **socket_context, + enum MHD_ConnectionNotificationCode toe) { + if (toe == MHD_CONNECTION_NOTIFY_STARTED) { + fprintf(stderr, "[CPP-V3] [MHD-CONNECT] New connection started\n"); + } else if (toe == MHD_CONNECTION_NOTIFY_CLOSED) { + fprintf(stderr, "[CPP-V3] [MHD-DISCONNECT] Connection closed\n"); + } + return MHD_YES; +} + +int main() { + Aws::SDKOptions options; + + // Configure AWS SDK logging to output to stderr (which goes to server.log) + // Using Debug level to capture all SDK activity including CryptoModule errors + options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; + options.loggingOptions.logger_create_fn = []() { + return std::make_shared( + Aws::Utils::Logging::LogLevel::Debug + ); + }; + + fprintf(stderr, "[CONFIG] AWS SDK logging enabled at Debug level\n"); + + Aws::InitAPI(options); + + // Detect CPU core count and configure threading + unsigned int num_cores = std::thread::hardware_concurrency(); + if (num_cores == 0) { + num_cores = 4; // Fallback if detection fails + fprintf(stderr, "[WARNING] CPU core detection failed, defaulting to %u cores\n", num_cores); + } + + // Thread pool size = num_cores * 2 (allows for I/O blocking without starving throughput) + g_thread_pool_size = num_cores * 2; + unsigned int connection_limit = g_thread_pool_size; + + // Log configuration + fprintf(stderr, "[CONFIG] Detected CPU cores: %u\n", num_cores); + fprintf(stderr, "[CONFIG] Thread pool size: %u\n", g_thread_pool_size); + fprintf(stderr, "[CONFIG] Connection limit: %u\n", connection_limit); + fprintf(stderr, "[CONFIG] Each S3 client will use 512 max connections\n"); + + int port = 8091; + + struct MHD_Daemon *daemon = + MHD_start_daemon(MHD_USE_POLL_INTERNALLY | MHD_USE_INTERNAL_POLLING_THREAD | MHD_USE_ERROR_LOG, + port, NULL, NULL, + &request_handler, NULL, + MHD_OPTION_EXTERNAL_LOGGER, log_mhd_error, NULL, + MHD_OPTION_NOTIFY_CONNECTION, notify_connection, NULL, + MHD_OPTION_NOTIFY_COMPLETED, request_completed, NULL, + MHD_OPTION_THREAD_POOL_SIZE, g_thread_pool_size, + MHD_OPTION_CONNECTION_LIMIT, connection_limit, + MHD_OPTION_CONNECTION_TIMEOUT, 10, + MHD_OPTION_END); + + if (!daemon) { + fprintf(stderr, "Failed to start server on port %d\n", port); + Aws::ShutdownAPI(options); + return 1; + } + + fprintf(stderr, "Server running on port %d\n", port); + sleep(10000); + + MHD_stop_daemon(daemon); + Aws::ShutdownAPI(options); + fprintf(stderr, "Ending server\n"); + return 0; +} diff --git a/test-server/go-v3-transition-server/.duvet/.gitignore b/test-server/go-v3-transition-server/.duvet/.gitignore new file mode 100644 index 00000000..32ad579b --- /dev/null +++ b/test-server/go-v3-transition-server/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ diff --git a/test-server/go-v3-transition-server/.duvet/config.toml b/test-server/go-v3-transition-server/.duvet/config.toml new file mode 100644 index 00000000..713e72d3 --- /dev/null +++ b/test-server/go-v3-transition-server/.duvet/config.toml @@ -0,0 +1,27 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "local-go-s3ec/v4/**/*.go" + +# Include required specifications here +[[specification]] +source = "../specification/s3-encryption/client.md" +[[specification]] +source = "../specification/s3-encryption/decryption.md" +[[specification]] +source = "../specification/s3-encryption/encryption.md" +[[specification]] +source = "../specification/s3-encryption/key-commitment.md" +[[specification]] +source = "../specification/s3-encryption/key-derivation.md" +[[specification]] +source = "../specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "../specification/s3-encryption/data-format/metadata-strategy.md" + +[report.html] +enabled = true + +# Enable snapshots to prevent requirement coverage regressions +[report.snapshot] +enabled = false diff --git a/test-server/go-v3-transition-server/Makefile b/test-server/go-v3-transition-server/Makefile new file mode 100644 index 00000000..a254acdf --- /dev/null +++ b/test-server/go-v3-transition-server/Makefile @@ -0,0 +1,39 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: build-server start-server stop-server wait-for-server + +PID_FILE := server.pid +PORT := 8095 + +build-server: + @echo "Building Go V3 Transition server..." + go mod tidy + +start-server: + @echo "Starting Go V3 Transition server..." + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + go run . > server.log 2>&1 & echo $$! > $(PID_FILE) + @echo "Go V3 Transition server starting..." + +stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true + @if [ -f $(PID_FILE) ]; then \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ + fi + @rm -f server.log + @echo "Server stopped" + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT) + +duvet: + duvet report + +view-report-mac: + open .duvet/reports/report.html diff --git a/test-server/go-v3-transition-server/README.md b/test-server/go-v3-transition-server/README.md new file mode 100644 index 00000000..e7e226f7 --- /dev/null +++ b/test-server/go-v3-transition-server/README.md @@ -0,0 +1,23 @@ +# S3EC Go V3 Transition Test Server + +This is the Go implementation of the S3ECTestServer framework for S3EC Go V3 Transition. It provides a server implementation for testing Go S3 Encryption Client V3 Transition functionality. + +## Overview + +The S3EC Go test server implements the S3ECTestServer service defined in the shared Smithy model. It provides endpoints for: + +- Creating S3 Encryption Clients +- Putting objects with encryption +- Getting and decrypting objects + +## Usage + +To run the server: + +```console +go run . +``` + +This will start the server running on port `8095`. + +The server is used as part of the testing framework to verify cross-language compatibility of the S3 Encryption Client implementations. diff --git a/test-server/go-v3-transition-server/go.mod b/test-server/go-v3-transition-server/go.mod new file mode 100644 index 00000000..50f1259a --- /dev/null +++ b/test-server/go-v3-transition-server/go.mod @@ -0,0 +1,35 @@ +module github.com/aws/amazon-s3-encryption-client-python/test-server/go-server + +go 1.21 + +require ( + github.com/aws/amazon-s3-encryption-client-go/v3 v3.0.0 + github.com/aws/aws-sdk-go-v2 v1.24.0 + github.com/aws/aws-sdk-go-v2/config v1.26.1 + github.com/aws/aws-sdk-go-v2/service/kms v1.27.4 + github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5 + github.com/google/uuid v1.5.0 + github.com/gorilla/mux v1.8.1 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.16.12 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 // indirect + github.com/aws/smithy-go v1.19.0 // indirect +) + +// S3EC Go V4 is not released to pkg.go.dev as of writing. +// It is included as a submodule and referenced locally. +replace github.com/aws/amazon-s3-encryption-client-go/v3 => ./local-go-s3ec/v3 diff --git a/test-server/go-v3-transition-server/go.sum b/test-server/go-v3-transition-server/go.sum new file mode 100644 index 00000000..1bb969a3 --- /dev/null +++ b/test-server/go-v3-transition-server/go.sum @@ -0,0 +1,45 @@ + +github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk= +github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo= +github.com/aws/aws-sdk-go-v2/config v1.26.1 h1:z6DqMxclFGL3Zfo+4Q0rLnAZ6yVkzCRxhRMsiRQnD1o= +github.com/aws/aws-sdk-go-v2/config v1.26.1/go.mod h1:ZB+CuKHRbb5v5F0oJtGdhFTelmrxd4iWO1lf0rQwSAg= +github.com/aws/aws-sdk-go-v2/credentials v1.16.12 h1:v/WgB8NxprNvr5inKIiVVrXPuuTegM+K8nncFkr1usU= +github.com/aws/aws-sdk-go-v2/credentials v1.16.12/go.mod h1:X21k0FjEJe+/pauud82HYiQbEr9jRKY3kXEIQ4hXeTQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10/go.mod h1:K2WGI7vUvkIv1HoNbfBA1bvIZ+9kL3YVmWxeKuLQsiw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 h1:v+HbZaCGmOwnTTVS86Fleq0vPzOd7tnJGbFhP0stNLs= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9/go.mod h1:Xjqy+Nyj7VDLBtCMkQYOw1QYfAEZCVLrfI0ezve8wd4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 h1:N94sVhRACtXyVcjXxrwK1SKFIJrA9pOJ5yu2eSHnmls= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9/go.mod h1:hqamLz7g1/4EJP+GH5NBhcUMLjW+gKLQabgyz6/7WAU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 h1:ugD6qzjYtB7zM5PN/ZIeaAIyefPaD82G8+SJopgvUpw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9/go.mod h1:YD0aYBWCrPENpHolhKw2XDlTIWae2GKXT1T4o6N6hiM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 h1:/90OR2XbSYfXucBMJ4U14wrjlfleq/0SB6dZDPncgmo= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9/go.mod h1:dN/Of9/fNZet7UrQQ6kTDo/VSwKPIq94vjlU16bRARc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 h1:Nf2sHxjMJR8CSImIVCONRi4g0Su3J+TSTbS7G0pUeMU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9/go.mod h1:idky4TER38YIjr2cADF1/ugFMKvZV7p//pVeV5LZbF0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 h1:iEAeF6YC3l4FzlJPP9H3Ko1TXpdjdqWffxXjp8SY6uk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9/go.mod h1:kjsXoK23q9Z/tLBrckZLLyvjhZoS+AGrzqzUfEClvMM= +github.com/aws/aws-sdk-go-v2/service/kms v1.27.4 h1:c75pHGBV3h6WOsIjbJhLyOnlCPXzap45nbiP2Z5jk5M= +github.com/aws/aws-sdk-go-v2/service/kms v1.27.4/go.mod h1:D9FVDkZjkZnnFHymJ3fPVz0zOUlNSd0xcIIVmmrAac8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5 h1:Keso8lIOS+IzI2MkPZyK6G0LYcK3My2LQ+T5bxghEAY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5/go.mod h1:vADO6Jn+Rq4nDtfwNjhgR84qkZwiC6FqCaXdw/kYwjA= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 h1:ldSFWz9tEHAwHNmjx2Cvy1MjP5/L9kNoR0skc6wyOOM= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.5/go.mod h1:CaFfXLYL376jgbP7VKC96uFcU8Rlavak0UlAwk1Dlhc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 h1:2k9KmFawS63euAkY4/ixVNsYYwrwnd5fIvgEKkfZFNM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5/go.mod h1:W+nd4wWDVkSUIox9bacmkBP5NMFQeTJ/xqNabpzSR38= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 h1:5UYvv8JUvllZsRnfrcMQ+hJ9jNICmcgKPAO1CER25Wg= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.5/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU= +github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= +github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= diff --git a/test-server/go-v3-transition-server/local-go-s3ec b/test-server/go-v3-transition-server/local-go-s3ec new file mode 160000 index 00000000..bf8a12f6 --- /dev/null +++ b/test-server/go-v3-transition-server/local-go-s3ec @@ -0,0 +1 @@ +Subproject commit bf8a12f61694d750a13a44f0a691dd7ced0ff904 diff --git a/test-server/go-v3-transition-server/main.go b/test-server/go-v3-transition-server/main.go new file mode 100644 index 00000000..64556f12 --- /dev/null +++ b/test-server/go-v3-transition-server/main.go @@ -0,0 +1,385 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "strings" + "sync" + + "github.com/aws/amazon-s3-encryption-client-go/v3/client" + "github.com/aws/amazon-s3-encryption-client-go/v3/materials" + "github.com/aws/amazon-s3-encryption-client-go/v3/commitment" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/google/uuid" + "github.com/gorilla/mux" +) + +// Server represents the Go test server +type Server struct { + clientCache map[string]*client.S3EncryptionClientV3 + kmsClient *kms.Client + mu sync.RWMutex +} + +// CreateClientInput represents the input for creating a client +type CreateClientInput struct { + Config S3ECConfig `json:"config"` +} + +// CreateClientOutput represents the output for creating a client +type CreateClientOutput struct { + ClientID string `json:"clientId"` +} + +// S3ECConfig represents the S3 encryption client configuration +type S3ECConfig struct { + EnableLegacyUnauthenticatedModes bool `json:"enableLegacyUnauthenticatedModes"` + EnableDelayedAuthenticationMode bool `json:"enableDelayedAuthenticationMode"` + EnableLegacyWrappingAlgorithms bool `json:"enableLegacyWrappingAlgorithms"` + SetBufferSize int64 `json:"setBufferSize"` + KeyMaterial KeyMaterial `json:"keyMaterial"` + CommitmentPolicy string `json:"commitmentPolicy"` +} + +// KeyMaterial represents the key material for encryption +type KeyMaterial struct { + RSAKey []byte `json:"rsaKey"` + AESKey []byte `json:"aesKey"` + KMSKeyID string `json:"kmsKeyId"` +} + +// PutObjectOutput represents the output for put object operation +type PutObjectOutput struct { + Bucket string `json:"bucket"` + Key string `json:"key"` + Metadata []string `json:"metadata"` +} + +// ErrorResponse represents an error response +type ErrorResponse struct { + Type string `json:"__type"` + Message string `json:"message"` +} + +// NewServer creates a new server instance +func NewServer() (*Server, error) { + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion("us-west-2"), + config.WithRetryMaxAttempts(5), + config.WithRetryMode(aws.RetryModeAdaptive), + ) + if err != nil { + return nil, fmt.Errorf("failed to load AWS config: %w", err) + } + + return &Server{ + clientCache: make(map[string]*client.S3EncryptionClientV3), + kmsClient: kms.NewFromConfig(cfg), + }, nil +} + +// createGenericServerError creates a generic server error response +func (s *Server) createGenericServerError(w http.ResponseWriter, message string, statusCode int) { + // Echo error to console + log.Printf("[Go V3-Transition] GenericServerError: %s (Status: %d)", message, statusCode) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + json.NewEncoder(w).Encode(ErrorResponse{ + Type: "software.amazon.encryption.s3#GenericServerError", + Message: message, + }) +} + +// createS3EncryptionClientError creates an S3 encryption client error response +func (s *Server) createS3EncryptionClientError(w http.ResponseWriter, message string, statusCode int) { + // Echo error to console + log.Printf("[Go V3-Transition] S3EncryptionClientError: %s (Status: %d)", message, statusCode) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + json.NewEncoder(w).Encode(ErrorResponse{ + Type: "software.amazon.encryption.s3#S3EncryptionClientError", + Message: message, + }) +} + +// metadataStringToMap converts metadata string to map +func metadataStringToMap(mdString string) (map[string]string, error) { + md := make(map[string]string) + if mdString == "" { + return md, nil + } + + mdList := strings.Split(mdString, ",") + for _, entry := range mdList { + // Split on "]:[" to separate key and value + parts := strings.Split(entry, "]:[") + if len(parts) == 2 { + // Remove remaining brackets from start and end + key := parts[0][1:] // Remove first character + value := parts[1][:len(parts[1])-1] // Remove last character + md[key] = value + } else { + return nil, fmt.Errorf("malformed metadata list entry: %s", entry) + } + } + return md, nil +} + +// createClient handles POST /client +func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { + // Read body + body, err := io.ReadAll(r.Body) + if err != nil { + s.createGenericServerError(w, "Failed to read request body", http.StatusBadRequest) + return + } + + var input CreateClientInput + if err := json.Unmarshal(body, &input); err != nil { + s.createGenericServerError(w, "Invalid JSON in request body", http.StatusBadRequest) + return + } + + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion("us-west-2"), + config.WithRetryMaxAttempts(5), + config.WithRetryMode(aws.RetryModeAdaptive), + ) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to load AWS config: %v", err), http.StatusInternalServerError) + return + } + + var commitmentPolicy commitment.CommitmentPolicy + switch input.Config.CommitmentPolicy { + case "REQUIRE_ENCRYPT_REQUIRE_DECRYPT": + commitmentPolicy = commitment.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + case "REQUIRE_ENCRYPT_ALLOW_DECRYPT": + commitmentPolicy = commitment.REQUIRE_ENCRYPT_ALLOW_DECRYPT + case "FORBID_ENCRYPT_ALLOW_DECRYPT": + commitmentPolicy = commitment.FORBID_ENCRYPT_ALLOW_DECRYPT + } + + // Create KMS keyring + kmsClient := kms.NewFromConfig(cfg) + keyring := materials.NewKmsKeyring(kmsClient, input.Config.KeyMaterial.KMSKeyID, func(options *materials.KeyringOptions) { + options.EnableLegacyWrappingAlgorithms = input.Config.EnableLegacyWrappingAlgorithms + }) + cmm, err := materials.NewCryptographicMaterialsManager(keyring) + + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to create CMM: %v", err), http.StatusInternalServerError) + return + } + + // Create S3 encryption client + var s3EncryptionClient *client.S3EncryptionClientV3 + s3PlaintextClient := s3.NewFromConfig(cfg) + s3EncryptionClient, err = client.New(s3PlaintextClient, cmm, func(clientOptions *client.EncryptionClientOptions) { + if input.Config.CommitmentPolicy != "" { + clientOptions.CommitmentPolicy = commitmentPolicy + } + clientOptions.EnableLegacyUnauthenticatedModes = input.Config.EnableLegacyUnauthenticatedModes + }) + + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to create S3EC: %v", err), http.StatusInternalServerError) + return + } + + // Generate client ID + clientID := uuid.New().String() + + // Store client in cache (protected by mutex) + s.mu.Lock() + s.clientCache[clientID] = s3EncryptionClient + s.mu.Unlock() + + // Return response + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(CreateClientOutput{ + ClientID: clientID, + }) +} + +// putObject handles PUT /object/{bucket}/{key} +func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + bucket := vars["bucket"] + key := vars["key"] + + clientID := r.Header.Get("ClientID") + if clientID == "" { + s.createGenericServerError(w, "ClientID header is required", http.StatusBadRequest) + return + } + + // Get client from cache (protected by mutex) + s.mu.RLock() + client, exists := s.clientCache[clientID] + s.mu.RUnlock() + + if !exists { + s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) + return + } + + // Read body + body, err := io.ReadAll(r.Body) + if err != nil { + s.createGenericServerError(w, "Failed to read request body", http.StatusBadRequest) + return + } + + // Get metadata from header + metadataHeader := r.Header.Get("Content-Metadata") + encCtx, err := metadataStringToMap(metadataHeader) + + // Create context with encryption context + ctx := context.Background() + encryptionContext := context.WithValue(ctx, "EncryptionContext", encCtx) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to parse metadata: %v", err), http.StatusBadRequest) + return + } + + // Create put object input + putInput := &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: strings.NewReader(string(body)), + } + + // Add metadata if present + if len(encCtx) > 0 { + putInput.Metadata = encCtx + } + + // Make the put object request using the encryption client + _, err = client.PutObject(encryptionContext, putInput) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to put object: %v", err), http.StatusInternalServerError) + return + } + + log.Printf("[Go V3-Transition] PutObject SUCCESS: Bucket=%s, Key=%s", bucket, key) + + // Return response + w.Header().Set("Content-Type", "application/json") + resp := PutObjectOutput{ + Bucket: bucket, + Key: key, + Metadata: []string{}, // TODO: pass metadata back in response + } + json.NewEncoder(w).Encode(resp) +} + +// getObject handles GET /object/{bucket}/{key} +func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + bucket := vars["bucket"] + key := vars["key"] + + clientID := r.Header.Get("ClientID") + if clientID == "" { + s.createGenericServerError(w, "ClientID header is required", http.StatusBadRequest) + return + } + + // Get client from cache (protected by mutex) + s.mu.RLock() + client, exists := s.clientCache[clientID] + s.mu.RUnlock() + + if !exists { + s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) + return + } + + // Get metadata from header + metadataHeader := r.Header.Get("Content-Metadata") + encCtx, err := metadataStringToMap(metadataHeader) + + ctx := context.Background() + encryptionContext := context.WithValue(ctx, "EncryptionContext", encCtx) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to parse metadata: %v", err), http.StatusBadRequest) + return + } + + // Create get object input + getInput := &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + } + + // Make the get object request using the encryption client + result, err := client.GetObject(encryptionContext, getInput) + if err != nil { + errMsg := err.Error() + // Shim the S3EC error message to the error message expected by the test server. + // We don't want to change the S3EC error message but the test server expects a specific error message; + // This is the appropriate place to rewrite the error message. + if strings.Contains(errMsg, "to decrypt x-amz-cek-alg value `kms` you must enable legacyWrappingAlgorithms on the keyring") { + s.createS3EncryptionClientError(w, "Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms", http.StatusInternalServerError) + return + } + + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to get object: %v", err), http.StatusInternalServerError) + return + } + defer result.Body.Close() + + // Read the body + body, err := io.ReadAll(result.Body) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to read object body: %v", err), http.StatusInternalServerError) + return + } + + // Convert metadata to string format + var metadataList []string + if result.Metadata != nil { + for k, v := range result.Metadata { + metadataList = append(metadataList, fmt.Sprintf("%s=%s", k, v)) + } + } + + metadataStr := strings.Join(metadataList, ",") + + log.Printf("[Go V3-Transition] GetObject SUCCESS: Bucket=%s, Key=%s", bucket, key) + + // Set response headers + w.Header().Set("Content-Metadata", metadataStr) + + // Return the body as response + w.Write(body) +} + +func main() { + server, err := NewServer() + if err != nil { + log.Fatalf("[Go V3-Transition] Failed to create Go V3 Transition server: %v", err) + } + + r := mux.NewRouter() + + // Register routes + r.HandleFunc("/client", server.createClient).Methods("POST") + r.HandleFunc("/object/{bucket}/{key}", server.putObject).Methods("PUT") + r.HandleFunc("/object/{bucket}/{key}", server.getObject).Methods("GET") + + fmt.Println("[Go V3-Transition] Starting Go V3 Transition server on :8095...") + log.Fatal(http.ListenAndServe(":8095", r)) +} diff --git a/test-server/go-v4-server/.duvet/.gitignore b/test-server/go-v4-server/.duvet/.gitignore new file mode 100644 index 00000000..93956e36 --- /dev/null +++ b/test-server/go-v4-server/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ \ No newline at end of file diff --git a/test-server/go-v4-server/.duvet/config.toml b/test-server/go-v4-server/.duvet/config.toml new file mode 100644 index 00000000..713e72d3 --- /dev/null +++ b/test-server/go-v4-server/.duvet/config.toml @@ -0,0 +1,27 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "local-go-s3ec/v4/**/*.go" + +# Include required specifications here +[[specification]] +source = "../specification/s3-encryption/client.md" +[[specification]] +source = "../specification/s3-encryption/decryption.md" +[[specification]] +source = "../specification/s3-encryption/encryption.md" +[[specification]] +source = "../specification/s3-encryption/key-commitment.md" +[[specification]] +source = "../specification/s3-encryption/key-derivation.md" +[[specification]] +source = "../specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "../specification/s3-encryption/data-format/metadata-strategy.md" + +[report.html] +enabled = true + +# Enable snapshots to prevent requirement coverage regressions +[report.snapshot] +enabled = false diff --git a/test-server/go-v4-server/Makefile b/test-server/go-v4-server/Makefile new file mode 100644 index 00000000..6c549db2 --- /dev/null +++ b/test-server/go-v4-server/Makefile @@ -0,0 +1,39 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: build-server start-server stop-server wait-for-server + +PID_FILE := server.pid +PORT := 8089 + +build-server: + @echo "Building Go V4 server..." + go mod tidy + +start-server: + @echo "Starting Go V4 server..." + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + go run . > server.log 2>&1 & echo $$! > $(PID_FILE) + @echo "Go V4 server starting..." + +stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true + @if [ -f $(PID_FILE) ]; then \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ + fi + @rm -f server.log + @echo "Server stopped" + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT) + +duvet: + duvet report + +view-report-mac: + open .duvet/reports/report.html diff --git a/test-server/go-v4-server/README.md b/test-server/go-v4-server/README.md new file mode 100644 index 00000000..d97a37bf --- /dev/null +++ b/test-server/go-v4-server/README.md @@ -0,0 +1,23 @@ +# S3EC Go V4 Test Server + +This is the Go implementation of the S3ECTestServer framework for S3EC Go V4. It provides a server implementation for testing Go S3 Encryption Client V4 functionality. + +## Overview + +The S3EC Go test server implements the S3ECTestServer service defined in the shared Smithy model. It provides endpoints for: + +- Creating S3 Encryption Clients +- Putting objects with encryption +- Getting and decrypting objects + +## Usage + +To run the server: + +```console +go run . +``` + +This will start the server running on port `8089`. + +The server is used as part of the testing framework to verify cross-language compatibility of the S3 Encryption Client implementations. diff --git a/test-server/go-v4-server/go.mod b/test-server/go-v4-server/go.mod new file mode 100644 index 00000000..33b1cc9f --- /dev/null +++ b/test-server/go-v4-server/go.mod @@ -0,0 +1,35 @@ +module github.com/aws/amazon-s3-encryption-client-python/test-server/go-server + +go 1.24 + +require ( + github.com/aws/amazon-s3-encryption-client-go/v4 v4.0.0 + github.com/aws/aws-sdk-go-v2 v1.24.0 + github.com/aws/aws-sdk-go-v2/config v1.26.1 + github.com/aws/aws-sdk-go-v2/service/kms v1.27.4 + github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5 + github.com/google/uuid v1.5.0 + github.com/gorilla/mux v1.8.1 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.16.12 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 // indirect + github.com/aws/smithy-go v1.19.0 // indirect +) + +// S3EC Go V4 is not released to pkg.go.dev as of writing. +// It is included as a submodule and referenced locally. +replace github.com/aws/amazon-s3-encryption-client-go/v4 => ./local-go-s3ec/v4 diff --git a/test-server/go-v4-server/go.sum b/test-server/go-v4-server/go.sum new file mode 100644 index 00000000..f4e3646a --- /dev/null +++ b/test-server/go-v4-server/go.sum @@ -0,0 +1,44 @@ +github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk= +github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo= +github.com/aws/aws-sdk-go-v2/config v1.26.1 h1:z6DqMxclFGL3Zfo+4Q0rLnAZ6yVkzCRxhRMsiRQnD1o= +github.com/aws/aws-sdk-go-v2/config v1.26.1/go.mod h1:ZB+CuKHRbb5v5F0oJtGdhFTelmrxd4iWO1lf0rQwSAg= +github.com/aws/aws-sdk-go-v2/credentials v1.16.12 h1:v/WgB8NxprNvr5inKIiVVrXPuuTegM+K8nncFkr1usU= +github.com/aws/aws-sdk-go-v2/credentials v1.16.12/go.mod h1:X21k0FjEJe+/pauud82HYiQbEr9jRKY3kXEIQ4hXeTQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10/go.mod h1:K2WGI7vUvkIv1HoNbfBA1bvIZ+9kL3YVmWxeKuLQsiw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 h1:v+HbZaCGmOwnTTVS86Fleq0vPzOd7tnJGbFhP0stNLs= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9/go.mod h1:Xjqy+Nyj7VDLBtCMkQYOw1QYfAEZCVLrfI0ezve8wd4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 h1:N94sVhRACtXyVcjXxrwK1SKFIJrA9pOJ5yu2eSHnmls= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9/go.mod h1:hqamLz7g1/4EJP+GH5NBhcUMLjW+gKLQabgyz6/7WAU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 h1:ugD6qzjYtB7zM5PN/ZIeaAIyefPaD82G8+SJopgvUpw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9/go.mod h1:YD0aYBWCrPENpHolhKw2XDlTIWae2GKXT1T4o6N6hiM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 h1:/90OR2XbSYfXucBMJ4U14wrjlfleq/0SB6dZDPncgmo= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9/go.mod h1:dN/Of9/fNZet7UrQQ6kTDo/VSwKPIq94vjlU16bRARc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 h1:Nf2sHxjMJR8CSImIVCONRi4g0Su3J+TSTbS7G0pUeMU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9/go.mod h1:idky4TER38YIjr2cADF1/ugFMKvZV7p//pVeV5LZbF0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 h1:iEAeF6YC3l4FzlJPP9H3Ko1TXpdjdqWffxXjp8SY6uk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9/go.mod h1:kjsXoK23q9Z/tLBrckZLLyvjhZoS+AGrzqzUfEClvMM= +github.com/aws/aws-sdk-go-v2/service/kms v1.27.4 h1:c75pHGBV3h6WOsIjbJhLyOnlCPXzap45nbiP2Z5jk5M= +github.com/aws/aws-sdk-go-v2/service/kms v1.27.4/go.mod h1:D9FVDkZjkZnnFHymJ3fPVz0zOUlNSd0xcIIVmmrAac8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5 h1:Keso8lIOS+IzI2MkPZyK6G0LYcK3My2LQ+T5bxghEAY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5/go.mod h1:vADO6Jn+Rq4nDtfwNjhgR84qkZwiC6FqCaXdw/kYwjA= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 h1:ldSFWz9tEHAwHNmjx2Cvy1MjP5/L9kNoR0skc6wyOOM= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.5/go.mod h1:CaFfXLYL376jgbP7VKC96uFcU8Rlavak0UlAwk1Dlhc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 h1:2k9KmFawS63euAkY4/ixVNsYYwrwnd5fIvgEKkfZFNM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5/go.mod h1:W+nd4wWDVkSUIox9bacmkBP5NMFQeTJ/xqNabpzSR38= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 h1:5UYvv8JUvllZsRnfrcMQ+hJ9jNICmcgKPAO1CER25Wg= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.5/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU= +github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= +github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= diff --git a/test-server/go-v4-server/local-go-s3ec b/test-server/go-v4-server/local-go-s3ec new file mode 160000 index 00000000..bf8a12f6 --- /dev/null +++ b/test-server/go-v4-server/local-go-s3ec @@ -0,0 +1 @@ +Subproject commit bf8a12f61694d750a13a44f0a691dd7ced0ff904 diff --git a/test-server/go-v4-server/main.go b/test-server/go-v4-server/main.go new file mode 100644 index 00000000..50999e95 --- /dev/null +++ b/test-server/go-v4-server/main.go @@ -0,0 +1,385 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "strings" + "sync" + + "github.com/aws/amazon-s3-encryption-client-go/v4/client" + "github.com/aws/amazon-s3-encryption-client-go/v4/materials" + "github.com/aws/amazon-s3-encryption-client-go/v4/commitment" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/google/uuid" + "github.com/gorilla/mux" +) + +// Server represents the Go test server +type Server struct { + clientCache map[string]*client.S3EncryptionClientV4 + kmsClient *kms.Client + mu sync.RWMutex +} + +// CreateClientInput represents the input for creating a client +type CreateClientInput struct { + Config S3ECConfig `json:"config"` +} + +// CreateClientOutput represents the output for creating a client +type CreateClientOutput struct { + ClientID string `json:"clientId"` +} + +// S3ECConfig represents the S3 encryption client configuration +type S3ECConfig struct { + EnableLegacyUnauthenticatedModes bool `json:"enableLegacyUnauthenticatedModes"` + EnableDelayedAuthenticationMode bool `json:"enableDelayedAuthenticationMode"` + EnableLegacyWrappingAlgorithms bool `json:"enableLegacyWrappingAlgorithms"` + SetBufferSize int64 `json:"setBufferSize"` + KeyMaterial KeyMaterial `json:"keyMaterial"` + CommitmentPolicy string `json:"commitmentPolicy"` +} + +// KeyMaterial represents the key material for encryption +type KeyMaterial struct { + RSAKey []byte `json:"rsaKey"` + AESKey []byte `json:"aesKey"` + KMSKeyID string `json:"kmsKeyId"` +} + +// PutObjectOutput represents the output for put object operation +type PutObjectOutput struct { + Bucket string `json:"bucket"` + Key string `json:"key"` + Metadata []string `json:"metadata"` +} + +// ErrorResponse represents an error response +type ErrorResponse struct { + Type string `json:"__type"` + Message string `json:"message"` +} + +// NewServer creates a new server instance +func NewServer() (*Server, error) { + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion("us-west-2"), + config.WithRetryMaxAttempts(5), + config.WithRetryMode(aws.RetryModeAdaptive), + ) + if err != nil { + return nil, fmt.Errorf("failed to load AWS config: %w", err) + } + + return &Server{ + clientCache: make(map[string]*client.S3EncryptionClientV4), + kmsClient: kms.NewFromConfig(cfg), + }, nil +} + +// createGenericServerError creates a generic server error response +func (s *Server) createGenericServerError(w http.ResponseWriter, message string, statusCode int) { + // Echo error to console + log.Printf("[Go V4] GenericServerError: %s (Status: %d)", message, statusCode) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + json.NewEncoder(w).Encode(ErrorResponse{ + Type: "software.amazon.encryption.s3#GenericServerError", + Message: message, + }) +} + +// createS3EncryptionClientError creates an S3 encryption client error response +func (s *Server) createS3EncryptionClientError(w http.ResponseWriter, message string, statusCode int) { + // Echo error to console + log.Printf("[Go V4] S3EncryptionClientError: %s (Status: %d)", message, statusCode) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + json.NewEncoder(w).Encode(ErrorResponse{ + Type: "software.amazon.encryption.s3#S3EncryptionClientError", + Message: message, + }) +} + +// metadataStringToMap converts metadata string to map +func metadataStringToMap(mdString string) (map[string]string, error) { + md := make(map[string]string) + if mdString == "" { + return md, nil + } + + mdList := strings.Split(mdString, ",") + for _, entry := range mdList { + // Split on "]:[" to separate key and value + parts := strings.Split(entry, "]:[") + if len(parts) == 2 { + // Remove remaining brackets from start and end + key := parts[0][1:] // Remove first character + value := parts[1][:len(parts[1])-1] // Remove last character + md[key] = value + } else { + return nil, fmt.Errorf("malformed metadata list entry: %s", entry) + } + } + return md, nil +} + +// createClient handles POST /client +func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { + // Read body + body, err := io.ReadAll(r.Body) + if err != nil { + s.createGenericServerError(w, "Failed to read request body", http.StatusBadRequest) + return + } + + var input CreateClientInput + if err := json.Unmarshal(body, &input); err != nil { + s.createGenericServerError(w, "Invalid JSON in request body", http.StatusBadRequest) + return + } + + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion("us-west-2"), + config.WithRetryMaxAttempts(5), + config.WithRetryMode(aws.RetryModeAdaptive), + ) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to load AWS config: %v", err), http.StatusInternalServerError) + return + } + + var commitmentPolicy commitment.CommitmentPolicy + switch input.Config.CommitmentPolicy { + case "REQUIRE_ENCRYPT_REQUIRE_DECRYPT": + commitmentPolicy = commitment.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + case "REQUIRE_ENCRYPT_ALLOW_DECRYPT": + commitmentPolicy = commitment.REQUIRE_ENCRYPT_ALLOW_DECRYPT + case "FORBID_ENCRYPT_ALLOW_DECRYPT": + commitmentPolicy = commitment.FORBID_ENCRYPT_ALLOW_DECRYPT + } + + // Create KMS keyring + kmsClient := kms.NewFromConfig(cfg) + keyring := materials.NewKmsKeyring(kmsClient, input.Config.KeyMaterial.KMSKeyID, func(options *materials.KeyringOptions) { + options.EnableLegacyWrappingAlgorithms = input.Config.EnableLegacyWrappingAlgorithms + }) + cmm, err := materials.NewCryptographicMaterialsManager(keyring) + + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to create CMM: %v", err), http.StatusInternalServerError) + return + } + + // Create S3 encryption client + var s3EncryptionClient *client.S3EncryptionClientV4 + s3PlaintextClient := s3.NewFromConfig(cfg) + s3EncryptionClient, err = client.New(s3PlaintextClient, cmm, func(clientOptions *client.EncryptionClientOptions) { + if input.Config.CommitmentPolicy != "" { + clientOptions.CommitmentPolicy = commitmentPolicy + } + clientOptions.EnableLegacyUnauthenticatedModes = input.Config.EnableLegacyUnauthenticatedModes + }) + + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to create S3EC: %v", err), http.StatusInternalServerError) + return + } + + // Generate client ID + clientID := uuid.New().String() + + // Store client in cache (protected by mutex) + s.mu.Lock() + s.clientCache[clientID] = s3EncryptionClient + s.mu.Unlock() + + // Return response + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(CreateClientOutput{ + ClientID: clientID, + }) +} + +// putObject handles PUT /object/{bucket}/{key} +func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + bucket := vars["bucket"] + key := vars["key"] + + clientID := r.Header.Get("ClientID") + if clientID == "" { + s.createGenericServerError(w, "ClientID header is required", http.StatusBadRequest) + return + } + + // Get client from cache (protected by mutex) + s.mu.RLock() + client, exists := s.clientCache[clientID] + s.mu.RUnlock() + + if !exists { + s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) + return + } + + // Read body + body, err := io.ReadAll(r.Body) + if err != nil { + s.createGenericServerError(w, "Failed to read request body", http.StatusBadRequest) + return + } + + // Get metadata from header + metadataHeader := r.Header.Get("Content-Metadata") + encCtx, err := metadataStringToMap(metadataHeader) + + // Create context with encryption context + ctx := context.Background() + encryptionContext := context.WithValue(ctx, "EncryptionContext", encCtx) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to parse metadata: %v", err), http.StatusBadRequest) + return + } + + // Create put object input + putInput := &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: strings.NewReader(string(body)), + } + + // Add metadata if present + if len(encCtx) > 0 { + putInput.Metadata = encCtx + } + + // Make the put object request using the encryption client + _, err = client.PutObject(encryptionContext, putInput) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to put object: %v", err), http.StatusInternalServerError) + return + } + + log.Printf("[Go V4] PutObject SUCCESS: Bucket=%s, Key=%s", bucket, key) + + // Return response + w.Header().Set("Content-Type", "application/json") + resp := PutObjectOutput{ + Bucket: bucket, + Key: key, + Metadata: []string{}, // TODO: pass metadata back in response + } + json.NewEncoder(w).Encode(resp) +} + +// getObject handles GET /object/{bucket}/{key} +func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + bucket := vars["bucket"] + key := vars["key"] + + clientID := r.Header.Get("ClientID") + if clientID == "" { + s.createGenericServerError(w, "ClientID header is required", http.StatusBadRequest) + return + } + + // Get client from cache (protected by mutex) + s.mu.RLock() + client, exists := s.clientCache[clientID] + s.mu.RUnlock() + + if !exists { + s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) + return + } + + // Get metadata from header + metadataHeader := r.Header.Get("Content-Metadata") + encCtx, err := metadataStringToMap(metadataHeader) + + ctx := context.Background() + encryptionContext := context.WithValue(ctx, "EncryptionContext", encCtx) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to parse metadata: %v", err), http.StatusBadRequest) + return + } + + // Create get object input + getInput := &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + } + + // Make the get object request using the encryption client + result, err := client.GetObject(encryptionContext, getInput) + if err != nil { + errMsg := err.Error() + // Shim the S3EC error message to the error message expected by the test server. + // We don't want to change the S3EC error message but the test server expects a specific error message; + // This is the appropriate place to rewrite the error message. + if strings.Contains(errMsg, "to decrypt x-amz-cek-alg value `kms` you must enable legacyWrappingAlgorithms on the keyring") { + s.createS3EncryptionClientError(w, "Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms", http.StatusInternalServerError) + return + } + + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to get object: %v", err), http.StatusInternalServerError) + return + } + defer result.Body.Close() + + // Read the body + body, err := io.ReadAll(result.Body) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to read object body: %v", err), http.StatusInternalServerError) + return + } + + // Convert metadata to string format + var metadataList []string + if result.Metadata != nil { + for k, v := range result.Metadata { + metadataList = append(metadataList, fmt.Sprintf("%s=%s", k, v)) + } + } + + metadataStr := strings.Join(metadataList, ",") + + log.Printf("[Go V4] GetObject SUCCESS: Bucket=%s, Key=%s", bucket, key) + + // Set response headers + w.Header().Set("Content-Metadata", metadataStr) + + // Return the body as response + w.Write(body) +} + +func main() { + server, err := NewServer() + if err != nil { + log.Fatalf("[Go V4] Failed to create Go V4 server: %v", err) + } + + r := mux.NewRouter() + + // Register routes + r.HandleFunc("/client", server.createClient).Methods("POST") + r.HandleFunc("/object/{bucket}/{key}", server.putObject).Methods("PUT") + r.HandleFunc("/object/{bucket}/{key}", server.getObject).Methods("GET") + + fmt.Println("[Go V4] Starting Go V4 server on :8089...") + log.Fatal(http.ListenAndServe(":8089", r)) +} diff --git a/test-server/java-server/gradle.properties b/test-server/java-server/gradle.properties deleted file mode 100644 index 08afce82..00000000 --- a/test-server/java-server/gradle.properties +++ /dev/null @@ -1,11 +0,0 @@ -# Smithy versions -smithyJavaVersion=[0,1] -smithyGradleVersion=1.1.0 -smithyVersion=[1,2] - -# Performance optimization settings -org.gradle.parallel=true -org.gradle.caching=true -org.gradle.daemon=true -org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -org.gradle.workers.max=4 diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java b/test-server/java-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java deleted file mode 100644 index d992c435..00000000 --- a/test-server/java-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java +++ /dev/null @@ -1,109 +0,0 @@ -package software.amazon.encryption.s3; - -import software.amazon.awssdk.core.traits.Trait; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.encryption.s3.S3EncryptionClient; -import software.amazon.encryption.s3.materials.AesKeyring; -import software.amazon.encryption.s3.materials.Keyring; -import software.amazon.encryption.s3.materials.KmsKeyring; -import software.amazon.encryption.s3.materials.PartialRsaKeyPair; -import software.amazon.encryption.s3.materials.RsaKeyring; -import software.amazon.smithy.java.core.schema.Schema; -import software.amazon.smithy.java.server.RequestContext; -import software.amazon.encryption.s3.model.CreateClientInput; -import software.amazon.encryption.s3.model.CreateClientOutput; -import software.amazon.encryption.s3.model.GenericServerError; -import software.amazon.encryption.s3.model.KeyMaterial; -import software.amazon.encryption.s3.service.CreateClientOperation; - -import javax.crypto.spec.SecretKeySpec; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.security.KeyFactory; -import java.security.NoSuchAlgorithmException; -import java.security.spec.InvalidKeySpecException; -import java.security.spec.PKCS8EncodedKeySpec; -import java.util.Arrays; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; - -public class CreateClientOperationImpl implements CreateClientOperation { - private Map clientCache_; - - public CreateClientOperationImpl(Map clientCache) { - clientCache_ = clientCache; - } - - // Copied from S3EC. - private boolean onlyOneNonNull(Object... values) { - boolean haveOneNonNull = false; - for (Object o : values) { - if (o != null) { - if (haveOneNonNull) { - return false; - } - - haveOneNonNull = true; - } - } - - return haveOneNonNull; - } - - @Override - public CreateClientOutput createClient(CreateClientInput input, RequestContext context) { - try { - KeyMaterial key = input.getConfig().getKeyMaterial(); - if (!onlyOneNonNull(key.getAesKey(), key.getKmsKeyId(), key.getRsaKey())) { - throw new RuntimeException("KeyMaterial must be only one, non-null input!"); - } - Keyring keyring; - if (key.getAesKey() != null) { - byte[] keyBytes = new byte[key.getAesKey().remaining()]; - key.getAesKey().get(keyBytes); - keyring = AesKeyring.builder() - .wrappingKey(new SecretKeySpec(keyBytes, "AES")) - .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) - .build(); - } else if (key.getRsaKey() != null) { - try { - byte[] keyBytes = new byte[key.getRsaKey().remaining()]; - key.getRsaKey().get(keyBytes); - PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(keyBytes); - KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - keyring = RsaKeyring.builder() - .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) - .wrappingKeyPair(PartialRsaKeyPair.builder() - .privateKey(keyFactory.generatePrivate(keySpec)).build()) - .build(); - } catch (NoSuchAlgorithmException | InvalidKeySpecException nse) { - throw new RuntimeException(nse); - } - } else if (key.getKmsKeyId() != null) { - keyring = KmsKeyring.builder() - .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) - .wrappingKeyId(key.getKmsKeyId()) - .build(); - } else { - throw new RuntimeException("No KeyMaterial found!"); - } - S3Client s3Client = S3EncryptionClient.builder() - .keyring(keyring) - .build(); - UUID uuid = UUID.randomUUID(); - String uuidString = uuid.toString(); - clientCache_.put(uuidString, s3Client); - return CreateClientOutput.builder() - .clientId(uuidString) - .build(); - } catch (Exception e) { - StringWriter sw = new StringWriter(); - e.printStackTrace(new PrintWriter(sw)); - String stackTrace = sw.toString(); - throw GenericServerError.builder() - .message(stackTrace) - .build(); - } - } -} diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java b/test-server/java-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java deleted file mode 100644 index e7c5493f..00000000 --- a/test-server/java-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java +++ /dev/null @@ -1,72 +0,0 @@ -package software.amazon.encryption.s3; - -import software.amazon.awssdk.core.ResponseBytes; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.encryption.s3.S3EncryptionClientException; -import software.amazon.smithy.java.server.RequestContext; -import software.amazon.encryption.s3.model.GenericServerError; -import software.amazon.encryption.s3.model.GetObjectInput; -import software.amazon.encryption.s3.model.GetObjectOutput; -import software.amazon.encryption.s3.model.S3EncryptionClientError; -import software.amazon.encryption.s3.service.GetObjectOperation; - -import java.io.PrintWriter; -import java.io.StringWriter; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static software.amazon.encryption.s3.S3EncryptionClient.withAdditionalConfiguration; -import static software.amazon.encryption.s3.MetadataUtils.metadataListToMap; -import static software.amazon.encryption.s3.MetadataUtils.metadataMapToList; - -public class GetObjectOperationImpl implements GetObjectOperation { - private Map clientCache_; - public GetObjectOperationImpl(Map clientCache) { - clientCache_ = clientCache; - } - @Override - public GetObjectOutput getObject(GetObjectInput input, RequestContext context) { - try { - S3Client s3Client = clientCache_.get(input.getClientID()); - Map ecMap = metadataListToMap(input.getMetadata()); - - try { - ResponseBytes resp = s3Client.getObjectAsBytes(builder -> builder - .bucket(input.getBucket()) - .key(input.getKey()) - .overrideConfiguration(withAdditionalConfiguration(ecMap))); - - List mdAsList = metadataMapToList(resp.response().metadata()); - // Can't use asBB else it gets mad bc cant access backing array - ByteBuffer bb = ByteBuffer.wrap(resp.asByteArray()); - GetObjectOutput output = GetObjectOutput.builder() - .body(bb) - .metadata(mdAsList) - .build(); - return output; - } catch (S3EncryptionClientException s3EncryptionClientException) { - // Modeled exceptions MUST be returned as such - StringWriter sw = new StringWriter(); - s3EncryptionClientException.printStackTrace(new PrintWriter(sw)); - String stackTrace = sw.toString(); - throw S3EncryptionClientError.builder() - .message(stackTrace) - .build(); - } - } catch (Exception e) { - // Don't wrap modeled errors - if (e instanceof S3EncryptionClientError) { - throw e; - } - StringWriter sw = new StringWriter(); - e.printStackTrace(new PrintWriter(sw)); - String stackTrace = sw.toString(); - throw GenericServerError.builder() - .message(stackTrace) - .build(); - } - } -} diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java b/test-server/java-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java deleted file mode 100644 index 036289ec..00000000 --- a/test-server/java-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java +++ /dev/null @@ -1,43 +0,0 @@ -package software.amazon.encryption.s3; - -import software.amazon.encryption.s3.model.GenericServerError; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class MetadataUtils { - - /** - * Annoyingly, Smithy doesn't provide an interface for map types - * in HTTP headers, so we have to do the serde ourselves - */ - public static List metadataMapToList(Map md) { - List mdAsList = new ArrayList<>(md.size()); - for (Map.Entry keyValue : md.entrySet()) { - mdAsList.add("[" + keyValue.getKey() + "]:[" + keyValue.getValue() + "]"); - } - return mdAsList; - } - - public static Map metadataListToMap(List mdList) { - Map md = new HashMap<>(); - for (String entry : mdList) { - // Split on "]:[" to separate key and value - String[] parts = entry.split("]:\\["); - if (parts.length == 2) { - // Remove remaining brackets from start and end - String key = parts[0].substring(1); - String value = parts[1].substring(0, parts[1].length() - 1); - md.put(key, value); - } else { - throw GenericServerError.builder() - .message("Malformed metadata list entry: " + entry) - .build(); - } - } - return md; - } - -} diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java b/test-server/java-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java deleted file mode 100644 index 4c772673..00000000 --- a/test-server/java-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java +++ /dev/null @@ -1,55 +0,0 @@ -package software.amazon.encryption.s3; - -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.PutObjectResponse; -import software.amazon.smithy.java.server.RequestContext; -import software.amazon.encryption.s3.model.GenericServerError; -import software.amazon.encryption.s3.model.PutObjectInput; -import software.amazon.encryption.s3.model.PutObjectOutput; -import software.amazon.encryption.s3.service.PutObjectOperation; - -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.Map; -import java.util.stream.Collectors; - -import static software.amazon.encryption.s3.S3EncryptionClient.withAdditionalConfiguration; -import static software.amazon.encryption.s3.MetadataUtils.metadataListToMap; - -public class PutObjectOperationImpl implements PutObjectOperation { - - private Map clientCache_; - - public PutObjectOperationImpl(Map clientCache) { - clientCache_ = clientCache; - } - - @Override - public PutObjectOutput putObject(PutObjectInput input, RequestContext context) { - try { - final Map metadata = metadataListToMap(input.getMetadata()); - S3Client s3Client = clientCache_.get(input.getClientID()); - s3Client.putObject(builder -> builder - .bucket(input.getBucket()) - .key(input.getKey()) - .overrideConfiguration(withAdditionalConfiguration(metadata)), - RequestBody.fromByteBuffer(input.getBody()) - ); - // The real S3 doesn't provide bucket/key/metadata, so Test doesn't need to either, but we do anyway - return PutObjectOutput.builder() - .bucket(input.getBucket()) - .key(input.getKey()) - .metadata(input.getMetadata()) - .build(); - } catch (Exception e) { - StringWriter sw = new StringWriter(); - e.printStackTrace(new PrintWriter(sw)); - String stackTrace = sw.toString(); - throw GenericServerError.builder() - .message(stackTrace) - .build(); - } - } -} diff --git a/test-server/java-tests/README.md b/test-server/java-tests/README.md index eee84863..2a9b80ee 100644 --- a/test-server/java-tests/README.md +++ b/test-server/java-tests/README.md @@ -1,8 +1,8 @@ -## Java Tests +# Java Tests This project contains Java client tests for the S3 Encryption Client. -### Running Tests +## Running Tests To run the integration tests for this project: diff --git a/test-server/java-tests/build.gradle.kts b/test-server/java-tests/build.gradle.kts index f35a2ac6..2d1cbdeb 100644 --- a/test-server/java-tests/build.gradle.kts +++ b/test-server/java-tests/build.gradle.kts @@ -16,7 +16,11 @@ dependencies { // Test dependencies testImplementation("org.junit.jupiter:junit-jupiter:5.13.0") testRuntimeOnly("org.junit.platform:junit-platform-launcher") + // JUnit Suite support for test ordering + testImplementation("org.junit.platform:junit-platform-suite-api:1.10.0") + testRuntimeOnly("org.junit.platform:junit-platform-suite-engine:1.10.0") testImplementation("com.amazonaws:aws-java-sdk:1.12.788") + testImplementation("software.amazon.awssdk:s3:2.37.1") testImplementation("org.bouncycastle:bcprov-jdk15on:1.70") } @@ -46,6 +50,30 @@ tasks { useJUnitPlatform() testClassesDirs = sourceSets["it"].output.classesDirs classpath = sourceSets["it"].runtimeClasspath + outputs.upToDateWhen { false } + outputs.cacheIf { false } + + // Enable parallel test execution + systemProperty("junit.jupiter.execution.parallel.enabled", "true") + systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") + systemProperty("junit.jupiter.execution.parallel.mode.classes.default", "concurrent") + // Configure thread pool size - adjust based on I/O-bound nature of tests + systemProperty("junit.jupiter.execution.parallel.config.strategy", "fixed") + maxParallelForks = 1 // One JVM + systemProperty("junit.jupiter.execution.parallel.config.fixed.parallelism", + Math.max(1, Runtime.getRuntime().availableProcessors() - 3).toString()) // Scale with CPU, reserve 3 cores + + // Passing information from Gradle into the tests so that we can filter our servers + systemProperty("test.filter.servers", System.getProperty("test.filter.servers")) + // For debugging + // // Enable System.out output + // testLogging { + // events("passed", "skipped", "failed", "standardOut", "standardError") + // showStandardStreams = true + // } + + // // Disable AWS SDK v1 deprecation warnings + // systemProperty("aws.java.v1.disableDeprecationAnnouncement", "true") } } diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/CBCDecryptTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/CBCDecryptTests.java new file mode 100644 index 00000000..093ba2ab --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/CBCDecryptTests.java @@ -0,0 +1,184 @@ +/* +* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +* SPDX-License-Identifier: Apache-2.0 +*/ + +package software.amazon.encryption.s3; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static software.amazon.encryption.s3.TestUtils.*; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import com.amazonaws.services.s3.model.KMSEncryptionMaterials; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.api.Nested; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.GetObjectInput; +import software.amazon.encryption.s3.model.GetObjectOutput; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.PutObjectInput; +import software.amazon.encryption.s3.model.S3ECConfig; +import software.amazon.encryption.s3.model.S3EncryptionClientError; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; + +import com.amazonaws.services.s3.AmazonS3Encryption; +import com.amazonaws.services.s3.AmazonS3EncryptionClient; +import com.amazonaws.services.s3.model.CryptoConfiguration; +import com.amazonaws.services.s3.model.CryptoMode; +import com.amazonaws.services.s3.model.CryptoStorageMode; +import software.amazon.encryption.s3.TestUtils.*; +import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; + +/** +* Exhaustive tests for S3 Encryption Client round-trip operations. +* These tests cover various combinations of client versions, commitment policies, and encryption modes. +* +* Tests are based on the exhaustive test matrix defined at: +* https://tiny.amazon.com/3xnzwczl/loopcloumicrpeyJ3 +* +* These tests deal with decrypting CBC messages +*/ + +class CBCDecryptTests { + private static String sharedObjectKey = appendTestSuffix("test-cbc-kms-v1-"); + private static KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + @BeforeAll + static void encryptCBCObject() { + // Create the object using the old client + // V1 Client + EncryptionMaterialsProvider materialsProvider = new KMSEncryptionMaterialsProvider(TestUtils.KMS_KEY_ARN); + + CryptoConfiguration v1Config = + new CryptoConfiguration(CryptoMode.EncryptionOnly) + .withStorageMode(CryptoStorageMode.ObjectMetadata) + .withAwsKmsRegion(TestUtils.KMS_REGION); + + AmazonS3Encryption v1Client = AmazonS3EncryptionClient.encryptionBuilder() + .withCryptoConfiguration(v1Config) + .withEncryptionMaterials(materialsProvider) + .build(); + + v1Client.putObject(TestUtils.BUCKET, sharedObjectKey, sharedObjectKey); + } + + @ParameterizedTest(name = "{0}: Transition configured with the default should decrypt CBC") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_the_default_should_decrypt_cbc(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .enableLegacyUnauthenticatedModes(true) + .enableLegacyWrappingAlgorithms(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, Arrays.asList(sharedObjectKey), EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should decrypt CBC") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_cbc(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .enableLegacyWrappingAlgorithms(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, Arrays.asList(sharedObjectKey), EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should decrypt CBC") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_cbc(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient decClient = TestUtils.testServerClientFor(language); + CreateClientOutput decClientOutput = decClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .enableLegacyWrappingAlgorithms(true) + .build()) + .build()); + String decS3ECId = decClientOutput.getClientId(); + + TestUtils.Decrypt(decClient, decS3ECId, Arrays.asList(sharedObjectKey), EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should decrypt CBC") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_allow_decrypt_should_decrypt_cbc(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient decClient = TestUtils.testServerClientFor(language); + CreateClientOutput decClientOutput = decClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) + .enableLegacyUnauthenticatedModes(true) + .enableLegacyWrappingAlgorithms(true) + .build()) + .build()); + String decS3ECId = decClientOutput.getClientId(); + + TestUtils.Decrypt(decClient, decS3ECId, Arrays.asList(sharedObjectKey), EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should fail to decrypt CBC") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_fail_to_decrypt_cbc(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient decClient = TestUtils.testServerClientFor(language); + CreateClientOutput decClientOutput = decClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String decS3ECId = decClientOutput.getClientId(); + + TestUtils.Decrypt_fails(decClient, decS3ECId, Arrays.asList(sharedObjectKey), EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Improved configured with the default should fail to decrypt CBC") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_the_default_should_fail_to_decrypt_cbc(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient decClient = TestUtils.testServerClientFor(language); + CreateClientOutput decClientOutput = decClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String decS3ECId = decClientOutput.getClientId(); + + TestUtils.Decrypt_fails(decClient, decS3ECId, Arrays.asList(sharedObjectKey), EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF); + } +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ExhaustiveRoundTripTests1_25.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ExhaustiveRoundTripTests1_25.java new file mode 100644 index 00000000..b161feb6 --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ExhaustiveRoundTripTests1_25.java @@ -0,0 +1,232 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.encryption.s3; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static software.amazon.encryption.s3.TestUtils.*; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import com.amazonaws.services.s3.model.KMSEncryptionMaterials; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.GetObjectInput; +import software.amazon.encryption.s3.model.GetObjectOutput; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.PutObjectInput; +import software.amazon.encryption.s3.model.S3ECConfig; +import software.amazon.encryption.s3.model.S3EncryptionClientError; + +import com.amazonaws.services.s3.AmazonS3Encryption; +import com.amazonaws.services.s3.AmazonS3EncryptionClient; +import com.amazonaws.services.s3.model.CryptoConfiguration; +import com.amazonaws.services.s3.model.CryptoMode; +import com.amazonaws.services.s3.model.CryptoStorageMode; +import software.amazon.encryption.s3.TestUtils.*; +import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; + +/** + * Exhaustive tests for S3 Encryption Client round-trip operations. + * These tests cover various combinations of client versions, commitment policies, and encryption modes. + * + * Tests are based on the exhaustive test matrix defined at: + * https://tiny.amazon.com/3xnzwczl/loopcloumicrpeyJ3 + * + * Tests 1-25 are included in this file. + */ +public class ExhaustiveRoundTripTests1_25 { + + @BeforeAll + public static void setup() { + TestUtils.validateServersRunning(); + } + + // Begin Exhaustive tests defined here: + // https://tiny.amazon.com/3xnzwczl/loopcloumicrpeyJ3 + + + // Exhaustive test 2 + // Outcome Version Operation Policy Content Encryption + // Pass Improved Decrypt ForbidEncryptAllowDecrypt CBC + + @ParameterizedTest(name = "{displayName} for Encrypt: Java-V1, Decrypt: {0}") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + public void GIVEN_CBCEncryptedData_AND_ImprovedClientDecryptingWithForbidEncryptAllowDecrypt_WHEN_Decrypt_THEN_Pass( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + final String objectKey = "test-key-kms-v1-" + language; + final String input = "simple-test-input"; + KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + // Create the object using the old client + // V1 Client + EncryptionMaterialsProvider materialsProvider = new KMSEncryptionMaterialsProvider(TestUtils.KMS_KEY_ARN); + + CryptoConfiguration v1Config = + new CryptoConfiguration(CryptoMode.AuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.ObjectMetadata) + .withAwsKmsRegion(TestUtils.KMS_REGION); + + AmazonS3Encryption v1Client = AmazonS3EncryptionClient.encryptionBuilder() + .withCryptoConfiguration(v1Config) + .withEncryptionMaterials(materialsProvider) + .build(); + + v1Client.putObject(TestUtils.BUCKET, objectKey, input); + + S3ECTestServerClient decClient = TestUtils.testServerClientFor(language); + CreateClientOutput decClientOutput = decClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyWrappingAlgorithms(true) + .build() + ) + .build()); + String decS3ECId = decClientOutput.getClientId(); + + // When: decrypt KC object with a current version client + GetObjectOutput output = decClient.getObject(GetObjectInput.builder() + .clientID(decS3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + + // Then: Pass + } + + // Exhaustive test 3 + // Outcome Version Operation Policy Content Encryption + // Pass Improved Decrypt ForbidEncryptAllowDecrypt GCM + + @ParameterizedTest(name = "{displayName} for Encrypt: Java-V1-GCM, Decrypt: {0}") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + public void GIVEN_GCMEncryptedData_AND_ImprovedClientDecryptingWithForbidEncryptAllowDecrypt_WHEN_Decrypt_THEN_Pass( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + final String objectKey = "test-key-kms-v1-gcm-" + language; + final String input = "simple-test-input"; + KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + CreateClientOutput output1 = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .enableLegacyWrappingAlgorithms(true) + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String s3ECId = output1.getClientId(); + + // Create the object using the old client with GCM encryption + // V1 Client with GCM + EncryptionMaterialsProvider materialsProvider = new KMSEncryptionMaterialsProvider(TestUtils.KMS_KEY_ARN); + + CryptoConfiguration v1Config = + new CryptoConfiguration(CryptoMode.StrictAuthenticatedEncryption) // StrictAuthenticatedEncryption uses GCM + .withStorageMode(CryptoStorageMode.ObjectMetadata) + .withAwsKmsRegion(TestUtils.KMS_REGION); + + AmazonS3Encryption v1Client = AmazonS3EncryptionClient.encryptionBuilder() + .withCryptoConfiguration(v1Config) + .withEncryptionMaterials(materialsProvider) + .build(); + + v1Client.putObject(TestUtils.BUCKET, objectKey, input); + + // When: decrypt GCM object with an improved version client + GetObjectOutput output = client.getObject(GetObjectInput.builder() + .clientID(s3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + + // Then: Pass + assertEquals(input, new String(output.getBody().array())); + } + + // Exhaustive test 4 + // Outcome Version Operation Policy Content Encryption + // Pass Improved Decrypt ForbidEncryptAllowDecrypt KC-GCM + + @ParameterizedTest(name = "{displayName} for Encrypt: {0}, Decrypt: {1}") + @MethodSource("software.amazon.encryption.s3.TestUtils#encryptImprovedDecryptImproved") + public void GIVEN_KCGCMEncryptedData_AND_ImprovedClientDecryptingWithForbidEncryptAllowDecrypt_WHEN_Decrypt_THEN_Pass( + TestUtils.LanguageServerTarget encLang, TestUtils.LanguageServerTarget decLang + ) throws Exception { + + S3ECTestServerClient encClient = TestUtils.testServerClientFor(encLang); + final String objectKey = "encrypt-kc-gcm-decrypt-improved-test-key-" + encLang + "-" + decLang; + final String input = "simple-test-input"; + KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + CreateClientOutput encClientOutput = encClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String encS3ECId = encClientOutput.getClientId(); + + // Given: object encrypted with key commitment + encClient.putObject(PutObjectInput.builder() + .clientID(encS3ECId) + .key(objectKey) + .bucket(TestUtils.BUCKET) + .body(ByteBuffer.wrap(input.getBytes(StandardCharsets.UTF_8))) + .build()); + + S3ECTestServerClient decClient = TestUtils.testServerClientFor(decLang); + CreateClientOutput decClientOutput = decClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String decS3ECId = decClientOutput.getClientId(); + + // At high concurrency, this test tends to get: + // BadDigest Message: The CRC64NVME you specified did not match the calculated checksum. + // I think this is a read after write issue. + // A better fix, would be to break this tests suite up into encrypt/decrypt + // rather than having a test for many pairs and doing encrypt/decrypt on each pair + Thread.sleep(100); + + // When: decrypt KC-GCM object with an improved version client with ForbidEncryptAllowDecrypt policy + GetObjectOutput output = decClient.getObject(GetObjectInput.builder() + .clientID(decS3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + + // Then: Pass + assertEquals(input, StandardCharsets.UTF_8.decode(output.getBody()).toString()); + } + +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTestSuite.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTestSuite.java new file mode 100644 index 00000000..ca495f56 --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTestSuite.java @@ -0,0 +1,258 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.encryption.s3; + +import static software.amazon.encryption.s3.TestUtils.*; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.S3ECConfig; + +/** + * GCM Test Suite + * + * This suite enforces execution order between GCM encrypt and decrypt phases: + * 1. EncryptTests - All encrypt tests run in parallel (within this phase) + * 2. DecryptTests - Waits for encrypt phase to complete, then all decrypt tests run in parallel + * + * Coordination is achieved using a CountDownLatch that EncryptTests signals upon completion + * and DecryptTests awaits before proceeding. + */ +public class GCMTestSuite { + // Synchronization latch - released when encrypt phase completes + private static final CountDownLatch encryptPhaseComplete = new CountDownLatch(1); + + /** + * GCM Encryption Tests - Encrypt Phase + * + * These tests encrypt objects using GCM (without key commitment) encryption algorithm. + * All tests in this class can run in parallel with each other. + * The encrypted objects are stored in thread-safe lists for use by DecryptTests. + */ + @Nested + @DisplayName("GCMTestSuite - Encrypt") + class EncryptTests { + private static final String sharedObjectKeyBase = "test-gcm-kms"; + private static final KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + // Thread-safe list for storing encrypted object keys + private static final List crossLanguageObjects = + Collections.synchronizedList(new ArrayList<>()); + + /** + * Public accessor for decrypt tests to retrieve encrypted object keys + */ + static List getCrossLanguageObjects() { + return new ArrayList<>(crossLanguageObjects); // Return defensive copy + } + + @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should encrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_forbid_encrypt_allow_decrypt_should_encrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), + crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Transition configured with the default should encrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_the_default_should_encrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), + crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should encrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_forbid_encrypt_allow_decrypt_should_encrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), + crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @AfterAll + static void signalEncryptionComplete() { + // Signal that all encryption tests have completed + encryptPhaseComplete.countDown(); + } + } + + /** + * GCM Decryption Tests - Decrypt Phase + * + * These tests decrypt objects that were encrypted by EncryptTests. + * All tests in this class can run fully in parallel with each other. + * They depend on EncryptTests completing first (enforced by @Order). + */ + @Nested + @DisplayName("GCMTestSuite - Decrypt") + class DecryptTests { + private static List crossLanguageObjects; + private static final KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + @BeforeAll + static void setup() throws InterruptedException { + // Wait for all encryption tests to complete + encryptPhaseComplete.await(); + + // Import encrypted objects from the encrypt phase + crossLanguageObjects = EncryptTests.getCrossLanguageObjects(); + + // Verify we have objects to decrypt + if (crossLanguageObjects.isEmpty()) { + throw new IllegalStateException( + "No encrypted objects found. Ensure EncryptTests runs first."); + } + } + + @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Transition configured with the default should decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_the_default_should_decrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_allow_decrypt_should_decrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should fail to decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_fail_to_decrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails(client, S3ECId, crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + } +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java new file mode 100644 index 00000000..18ebca47 --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java @@ -0,0 +1,1136 @@ +/* +* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +* SPDX-License-Identifier: Apache-2.0 +*/ + +package software.amazon.encryption.s3; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static software.amazon.encryption.s3.TestUtils.*; + +import java.nio.ByteBuffer; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import software.amazon.encryption.s3.TestUtils.LanguageServerTarget; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.*; + +/** +* Instruction File Failures Test Suite +* +* This suite enforces execution order between encrypt and decrypt phases: +* 1. EncryptTests - Encrypts objects with various key materials and creates test copies +* 2. DecryptTests - Waits for encrypt phase to complete, then tests decryption scenarios +* +* Coordination is achieved using a CountDownLatch that EncryptTests signals upon completion +* and DecryptTests awaits before proceeding. +* +*/ +public class InstructionFileFailures { + // Synchronization latch - released when encrypt phase completes + private static final CountDownLatch encryptPhaseComplete = new CountDownLatch(1); + + // Object key suffixes for test copies + private static final String SUFFIX_GOOD_COPY = "-good-copy"; + private static final String SUFFIX_BAD_BOTH_META_AND_INSTRUCTION = "-bad-both-meta-and-instruction"; + private static final String SUFFIX_BAD_ONLY_INSTRUCTION = "-bad-only-instruction"; + private static final String SUFFIX_BAD_JSON_INSTRUCTION = "-manipulated-bad-json-instruction"; + private static final String SUFFIX_MANIPULATED_INSTRUCTION = "-manipuldate-incorrect-key-instruction"; + + /** + * Encryption Tests - Encrypt Phase + * + * These tests encrypt objects using various key materials (KMS, RSA, AES) with instruction files. + * All tests in this class can run in parallel with each other. + * The encrypted objects are stored in thread-safe lists for use by DecryptTests. + */ + @Nested + @DisplayName("InstructionFileFailures - Encrypt") + class EncryptTests { + private static final String sharedObjectKeyBaseMetaDataMode = "test-instruction-files-cases"; + private static KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + // Thread-safe lists for storing encrypted object keys + private static final List crossLanguageObjectsKms = + Collections.synchronizedList(new ArrayList<>()); + private static final List crossLanguageObjectsRsa = + Collections.synchronizedList(new ArrayList<>()); + private static final List crossLanguageObjectsAes = + Collections.synchronizedList(new ArrayList<>()); + + // Thread-safe lists for envelope merge tests + private static final List crossLanguageObjectsMetadataOnly = + Collections.synchronizedList(new ArrayList<>()); + private static final List crossLanguageObjectsInstructionFileDeleted = + Collections.synchronizedList(new ArrayList<>()); + private static final List crossLanguageObjectsV3InstructionFileManipulated = + Collections.synchronizedList(new ArrayList<>()); + private static final List crossLanguageObjectsV2InstructionFileManipulated = + Collections.synchronizedList(new ArrayList<>()); + + private static KeyMaterial RSA_KEY; + private static KeyMaterial AES_KEY; + + @BeforeAll + static void setupKeys() throws Exception { + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(2048); + KeyPair keyPair = keyPairGen.generateKeyPair(); + + RSA_KEY = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(keyPair.getPrivate().getEncoded())) + .build(); + + KeyGenerator keyGen = KeyGenerator.getInstance("AES"); + keyGen.init(256); + SecretKey aesSecretKey = keyGen.generateKey(); + + AES_KEY = KeyMaterial.builder() + .aesKey(ByteBuffer.wrap(aesSecretKey.getEncoded())) + .build(); + } + + /** + * Public accessors for decrypt tests to retrieve encrypted object keys and key materials + */ + static List getCrossLanguageObjectsKms() { + return new ArrayList<>(crossLanguageObjectsKms); + } + + static List getCrossLanguageObjectsRsa() { + return new ArrayList<>(crossLanguageObjectsRsa); + } + + static List getCrossLanguageObjectsAes() { + return new ArrayList<>(crossLanguageObjectsAes); + } + + static KeyMaterial getRsaKey() { + return RSA_KEY; + } + + static KeyMaterial getAesKey() { + return AES_KEY; + } + + static KeyMaterial getKmsKeyArn() { + return kmsKeyArn; + } + + static List getCrossLanguageObjectsMetadataOnly() { + return new ArrayList<>(crossLanguageObjectsMetadataOnly); + } + + static List getCrossLanguageObjectsInstructionFileDeleted() { + return new ArrayList<>(crossLanguageObjectsInstructionFileDeleted); + } + + static List getCrossLanguageObjectsInstructionFileManipulatedV3() { + return new ArrayList<>(crossLanguageObjectsV3InstructionFileManipulated); + } + + static List getCrossLanguageObjectsInstructionFileManipulatedV2() { + return new ArrayList<>(crossLanguageObjectsV2InstructionFileManipulated); + } + + + public static Stream improvedClientsCanPutKMSWithInstructionFile() { + return improvedClientsForTest() + .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + + public static Stream improvedClientsCanPutRawRSAWithInstructionFile() { + return improvedClientsForTest() + .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + + public static Stream improvedClientsCanPutRawAESWithInstructionFile() { + return improvedClientsForTest() + .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + + @ParameterizedTest(name = "{0}: Encrypt KMS KC-GCM with instruction files") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$EncryptTests#improvedClientsCanPutKMSWithInstructionFile") + void encryptWithInstructionFilesKmsKcGcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .instructionFileConfig( + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() + ) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-kms" + language.getLanguageName()), + crossLanguageObjectsKms, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Encrypt RSA KC-GCM with instruction files") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$EncryptTests#improvedClientsCanPutRawRSAWithInstructionFile") + void encryptWithInstructionFilesRsaKcGcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .instructionFileConfig( + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() + ) + .build()) + .build()); + + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-rsa" + language.getLanguageName()), + crossLanguageObjectsRsa, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Encrypt AES KC-GCM with instruction files") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$EncryptTests#improvedClientsCanPutRawAESWithInstructionFile") + void encryptWithInstructionFilesAesKcGcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .instructionFileConfig( + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() + ) + .build()) + .build()); + + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-aes" + language.getLanguageName()), + crossLanguageObjectsAes, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Encrypt RSA KC-GCM metadata-only for envelope merge test") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$EncryptTests#improvedClientsCanPutRawRSAWithInstructionFile") + void encryptMetadataOnlyRsaKcGcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + // Encrypt with metadata-only (no instruction file) + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .build()) + .build()); + + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-envelope-merge-metadata-only-" + language.getLanguageName()), + crossLanguageObjectsMetadataOnly, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Encrypt RSA KC-GCM with instruction file for deletion test") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$EncryptTests#improvedClientsCanPutRawRSAWithInstructionFile") + void encryptWithInstructionFileForDeletionRsaKcGcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + // Encrypt with instruction file (will be deleted later) + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .instructionFileConfig( + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() + ) + .build()) + .build()); + + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-envelope-merge-instruction-deleted-" + language.getLanguageName()), + crossLanguageObjectsInstructionFileDeleted, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Encrypt KMS KC-GCM (V3) with instruction file for manipulation test") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$EncryptTests#improvedClientsCanPutKMSWithInstructionFile") + void encryptWithInstructionFileV3ForManipulationKmsKcGcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + // Encrypt with instruction file, will be manipulated later on. + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) + .instructionFileConfig( + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() + ) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-envelope-manipulation-instruction-" + language.getLanguageName()), + crossLanguageObjectsV3InstructionFileManipulated, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Encrypt KMS (V2) with instruction file for manipulation test") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$EncryptTests#improvedClientsCanPutKMSWithInstructionFile") + void encryptWithInstructionFileV2ForManipulationKms(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + // Encrypt with instruction file, will be manipulated later on. + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .instructionFileConfig( + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() + ) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-envelope-manipulation-instruction-" + language.getLanguageName()), + crossLanguageObjectsV2InstructionFileManipulated, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + ); + } + + + static void makeCopiesToVerifyThings() throws Exception { + // Create a plaintext S3 client to copy objects with instruction files + try (S3Client ptS3Client = S3Client.create()) { + List allCrossLanguageObjects = Stream.of( + crossLanguageObjectsKms.stream(), + crossLanguageObjectsRsa.stream(), + crossLanguageObjectsAes.stream() + ).flatMap(s -> s).collect(Collectors.toList()); + for (String objectKey : allCrossLanguageObjects) { + // Get the encrypted object + ResponseBytes encryptedObject = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + + // Get the instruction file + String instructionFileKey = objectKey + ".instruction"; + ResponseBytes instructionFile = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(TestUtils.BUCKET) + .key(instructionFileKey) + .build()); + + String instructionFileJson = instructionFile.asUtf8String(); + Map objectMetadata = encryptedObject.response().metadata(); + + // Put a strict copy, to verify that we know how to do this + putObjectWithInstructionFile( + ptS3Client, + objectKey + SUFFIX_GOOD_COPY, + encryptedObject.asByteArray(), + objectMetadata, + instructionFileJson + ); + + ObjectMapper mapper = new ObjectMapper(); + Map instructionFileMap = mapper.readValue(instructionFileJson, Map.class); + + instructionFileMap.put("x-amz-c", objectMetadata.get("x-amz-c")); + instructionFileMap.put("x-amz-d", objectMetadata.get("x-amz-d")); + instructionFileMap.put("x-amz-i", objectMetadata.get("x-amz-i")); + + String instructionFileWithCommitmentValues = mapper.writeValueAsString(instructionFileMap); + + // Put instruction files that should fail: + putObjectWithInstructionFile( + ptS3Client, + objectKey + SUFFIX_BAD_BOTH_META_AND_INSTRUCTION, + encryptedObject.asByteArray(), + objectMetadata, + instructionFileWithCommitmentValues + ); + + putObjectWithInstructionFile( + ptS3Client, + objectKey + SUFFIX_BAD_ONLY_INSTRUCTION, + encryptedObject.asByteArray(), + Map.of(), + instructionFileWithCommitmentValues + ); + + } + + // Delete instruction files for envelope merge tests + for (String objectKey : crossLanguageObjectsInstructionFileDeleted) { + String instructionFileKey = objectKey + ".instruction"; + try { + ptS3Client.deleteObject(builder -> builder + .bucket(TestUtils.BUCKET) + .key(instructionFileKey) + .build()); + } catch (Exception e) { + // Ignore if file doesn't exist + } + } + + // manipulate V3 instruction files + for (String objectKey: crossLanguageObjectsV3InstructionFileManipulated) { + // Get the encrypted object + ResponseBytes encryptedObject = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + + // Get the instruction file + String instructionFileKey = objectKey + ".instruction"; + ResponseBytes instructionFile = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(TestUtils.BUCKET) + .key(instructionFileKey) + .build()); + + String instructionFileJson = instructionFile.asUtf8String(); + Map objectMetadata = encryptedObject.response().metadata(); + + ObjectMapper mapper = new ObjectMapper(); + + Map invalidInstructionFileMap = new HashMap<>(); + invalidInstructionFileMap.put("invalid", "json"); + + String invalidInstructionFile = mapper.writeValueAsString(invalidInstructionFileMap); + + // Put instruction files that should fail: + putObjectWithInstructionFile( + ptS3Client, + objectKey + SUFFIX_BAD_JSON_INSTRUCTION + "-v3", + encryptedObject.asByteArray(), + objectMetadata, + invalidInstructionFile + ); + } + + // manipulate V2 instruction files + for (String objectKey: crossLanguageObjectsV2InstructionFileManipulated) { + // Get the encrypted object + ResponseBytes encryptedObject = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + + // Get the instruction file + String instructionFileKey = objectKey + ".instruction"; + ResponseBytes instructionFile = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(TestUtils.BUCKET) + .key(instructionFileKey) + .build()); + + String instructionFileJson = instructionFile.asUtf8String(); + Map objectMetadata = encryptedObject.response().metadata(); + + ObjectMapper mapper = new ObjectMapper(); + Map instructionFileMap = mapper.readValue(instructionFileJson, Map.class); + + instructionFileMap.put("x-amz-key-v2-tampered", instructionFileMap.get("x-amz-key-v2")); + instructionFileMap.remove("x-amz-key-v2"); + + String badKeyInstructionFile = mapper.writeValueAsString(instructionFileMap); + + // Put instruction files that should fail: + putObjectWithInstructionFile( + ptS3Client, + objectKey + SUFFIX_MANIPULATED_INSTRUCTION + "-v2", + encryptedObject.asByteArray(), + objectMetadata, + badKeyInstructionFile + ); + } + } + } + + static void putObjectWithInstructionFile( + S3Client ptS3Client, + String newObjectKey, + byte[] objectData, + Map objectMetadata, + String instructionFileJson + ) { + + // Put the encrypted object copy + ptS3Client.putObject(builder -> builder + .bucket(TestUtils.BUCKET) + .key(newObjectKey) + .metadata(objectMetadata) + .build(), + software.amazon.awssdk.core.sync.RequestBody.fromBytes(objectData)); + + // Put the instruction file copy + ptS3Client.putObject(builder -> builder + .bucket(TestUtils.BUCKET) + .key(newObjectKey + ".instruction") + .build(), + software.amazon.awssdk.core.sync.RequestBody.fromBytes(instructionFileJson.getBytes(java.nio.charset.StandardCharsets.UTF_8))); + } + + @AfterAll + static void signalEncryptionComplete() throws Exception { + makeCopiesToVerifyThings(); + + // Signal that all encryption tests have completed + encryptPhaseComplete.countDown(); + } + } + + /** + * Decryption Tests - Decrypt Phase + * + * These tests decrypt objects that were encrypted by EncryptTests. + * All tests in this class can run fully in parallel with each other. + * They depend on EncryptTests completing first. + */ + @Nested + @DisplayName("InstructionFileFailures - Decrypt") + class DecryptTests { + private static List crossLanguageObjectsKms; + private static List crossLanguageObjectsRsa; + private static List crossLanguageObjectsAes; + private static List crossLanguageObjectsMetadataOnly; + private static List crossLanguageObjectsInstructionFileDeleted; + private static List crossLanguageObjectsInstructionFileManipulatedV3; + private static List crossLanguageObjectsInstructionFileManipulatedV2; + private static KeyMaterial kmsKeyArn; + private static KeyMaterial RSA_KEY; + private static KeyMaterial AES_KEY; + + @BeforeAll + static void setup() throws InterruptedException { + // Wait for all encryption tests to complete + encryptPhaseComplete.await(); + + // Import encrypted objects and key materials from the encrypt phase + crossLanguageObjectsKms = EncryptTests.getCrossLanguageObjectsKms(); + crossLanguageObjectsRsa = EncryptTests.getCrossLanguageObjectsRsa(); + crossLanguageObjectsAes = EncryptTests.getCrossLanguageObjectsAes(); + crossLanguageObjectsMetadataOnly = EncryptTests.getCrossLanguageObjectsMetadataOnly(); + crossLanguageObjectsInstructionFileDeleted = EncryptTests.getCrossLanguageObjectsInstructionFileDeleted(); + crossLanguageObjectsInstructionFileManipulatedV3 = EncryptTests.getCrossLanguageObjectsInstructionFileManipulatedV3(); + crossLanguageObjectsInstructionFileManipulatedV2 = EncryptTests.getCrossLanguageObjectsInstructionFileManipulatedV2(); + kmsKeyArn = EncryptTests.getKmsKeyArn(); + RSA_KEY = EncryptTests.getRsaKey(); + AES_KEY = EncryptTests.getAesKey(); + + // Verify we have objects to decrypt + if (crossLanguageObjectsKms.isEmpty() && crossLanguageObjectsRsa.isEmpty() && crossLanguageObjectsAes.isEmpty()) { + throw new IllegalStateException( + "No encrypted objects found. Ensure EncryptTests runs first."); + } + } + + public static Stream clientsCanGetKMSWithInstructionFile() { + Stream improved = improvedClientsForTest() + .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + Stream transition = transitionClientsForTest() + .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + return Stream.concat(improved, transition); + } + + public static Stream clientsCanGetRawRSAWithInstructionFile() { + Stream improved = improvedClientsForTest() + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + Stream transition = transitionClientsForTest() + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + return Stream.concat(improved, transition); + } + + public static Stream clientsCanGetRawAESWithInstructionFile() { + Stream improved = improvedClientsForTest() + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + Stream transition = transitionClientsForTest() + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + return Stream.concat(improved, transition); + } + + // KMS instruction files decrypt + + @ParameterizedTest(name = "{0}: Successfully decrypt KMS encrypted original and good-copy objects") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") + void decryptKmsOriginalAndGoodCopyObjectsSucceeds(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsKms, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsKms + .stream() + .map(key -> key + SUFFIX_GOOD_COPY) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + crossLanguageObjectsKms + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt KMS when commitment is duplicated in metadata and instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") + void decryptKmsWithDuplicateCommitmentInMetadataAndInstructionFails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsKms + .stream() + .map(key -> key + SUFFIX_BAD_BOTH_META_AND_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt KMS when commitment is only in instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") + void decryptKmsWithCommitmentOnlyInInstructionFileFails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsKms + .stream() + .map(key -> key + SUFFIX_BAD_ONLY_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt KMS duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") + void decryptKmsWithDuplicateCommitmentFailsWithForbidPolicy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsKms + .stream() + .map(key -> key + SUFFIX_BAD_BOTH_META_AND_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt KMS instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") + void decryptKmsWithInstructionFileCommitmentFailsWithForbidPolicy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsKms + .stream() + .map(key -> key + SUFFIX_BAD_ONLY_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + // RSA instruction file decrypt + + @ParameterizedTest(name = "{0}: Successfully decrypt RSA encrypted original and good-copy objects") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") + void decryptRsaOriginalAndGoodCopyObjectsSucceeds(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsRsa, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + SUFFIX_GOOD_COPY) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + crossLanguageObjectsRsa + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt RSA when commitment is duplicated in metadata and instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") + void decryptRsaWithDuplicateCommitmentInMetadataAndInstructionFails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + SUFFIX_BAD_BOTH_META_AND_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt RSA when commitment is only in instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") + void decryptRsaWithCommitmentOnlyInInstructionFileFails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + SUFFIX_BAD_ONLY_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt RSA duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") + void decryptRsaWithDuplicateCommitmentFailsWithForbidPolicy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + SUFFIX_BAD_BOTH_META_AND_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt RSA instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") + void decryptRsaWithInstructionFileCommitmentFailsWithForbidPolicy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + SUFFIX_BAD_ONLY_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + // AES instruction file decrypt + + @ParameterizedTest(name = "{0}: Successfully decrypt AES encrypted original and good-copy objects") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") + void decryptAesOriginalAndGoodCopyObjectsSucceeds(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsAes, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + SUFFIX_GOOD_COPY) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + crossLanguageObjectsAes + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt AES when commitment is duplicated in metadata and instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") + void decryptAesWithDuplicateCommitmentInMetadataAndInstructionFails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + SUFFIX_BAD_BOTH_META_AND_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt AES when commitment is only in instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") + void decryptAesWithCommitmentOnlyInInstructionFileFails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + SUFFIX_BAD_ONLY_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt AES duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") + void decryptAesWithDuplicateCommitmentFailsWithForbidPolicy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + SUFFIX_BAD_BOTH_META_AND_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt AES instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") + void decryptAesWithInstructionFileCommitmentFailsWithForbidPolicy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + SUFFIX_BAD_ONLY_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + // Envelope merge tests + + @ParameterizedTest(name = "{0}: Successfully decrypt metadata-only object with instruction file config") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") + void decryptMetadataOnlyObjectWithInstructionFileConfigSucceeds(TestUtils.LanguageServerTarget language) { + if (crossLanguageObjectsMetadataOnly.isEmpty()) return; + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + // Configure client to look for instruction file but metadata has complete envelope + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .instructionFileConfig( + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() + ) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // Should succeed - instruction file doesn't exist but metadata has complete envelope + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsMetadataOnly, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt when metadata incomplete and instruction file deleted") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") + void decryptWithIncompleteMetadataAndNoInstructionFileFails(TestUtils.LanguageServerTarget language) { + if (crossLanguageObjectsInstructionFileDeleted.isEmpty()) return; + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + // Configure client for metadata-only but metadata is incomplete + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // Should fail - metadata incomplete (missing x-amz-3, x-amz-w), instruction file deleted + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsInstructionFileDeleted, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt with instruction file config when file deleted and metadata incomplete") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") + void decryptWithInstructionFileConfigWhenFileDeletedFails(TestUtils.LanguageServerTarget language) { + if (crossLanguageObjectsInstructionFileDeleted.isEmpty()) return; + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + // Configure client to look for instruction file but it's been deleted + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .instructionFileConfig( + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() + ) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // Should fail - instruction file deleted, metadata incomplete (missing x-amz-3, x-amz-w) + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsInstructionFileDeleted, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt with manipulated V3 Instruction File") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") + void decryptWithManipulatedInstructionFileV3ImprovedClients(TestUtils.LanguageServerTarget language) { + if (TRANSITION_VERSIONS.contains(language.getLanguageName())) { + return; + } + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsInstructionFileManipulatedV3 + .stream() + .map(key -> key + SUFFIX_BAD_JSON_INSTRUCTION + "-v3") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt with manipulated V2 Instruction File") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") + void decryptWithManipulatedInstructionFileV2ImprovedClients(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsInstructionFileManipulatedV2 + .stream() + .map(key -> key + SUFFIX_MANIPULATED_INSTRUCTION + "-v2") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + ); + } + } +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTestSuite.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTestSuite.java new file mode 100644 index 00000000..d256f909 --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTestSuite.java @@ -0,0 +1,391 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.encryption.s3; + +import static software.amazon.encryption.s3.TestUtils.*; + +import java.nio.ByteBuffer; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.opentest4j.TestAbortedException; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.InstructionFileConfig; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.S3ECConfig; + +/** + * KC-GCM Test Suite + * + * This suite enforces execution order between KC-GCM encrypt and decrypt phases: + * 1. EncryptTests - All encrypt tests run in parallel (within this phase) + * 2. DecryptTests - Waits for encrypt phase to complete, then all decrypt tests run in parallel + * + * Coordination is achieved using a CountDownLatch that EncryptTests signals upon completion + * and DecryptTests awaits before proceeding. + */ +public class KC_GCMTestSuite { + // Synchronization latch - released when encrypt phase completes + private static final CountDownLatch encryptPhaseComplete = new CountDownLatch(1); + + /** + * KC-GCM Encryption Tests - Encrypt Phase + * + * These tests encrypt objects using Key Commitment GCM encryption algorithm. + * All tests in this class can run in parallel with each other. + * The encrypted objects are stored in thread-safe lists for use by DecryptTests. + */ + @Nested + @DisplayName("KC_GCMTestSuite - Encrypt") + class EncryptTests { + private static final String sharedObjectKeyBaseMetaDataMode = "test-kc-gcm-kms"; + private static final String sharedObjectKeyBaseInsFileMode = "test-kc-gcm-rsa-instruction-file"; + private static final KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + // Thread-safe lists for storing encrypted object keys + private static final List crossLanguageObjectsMetaDataMode = + Collections.synchronizedList(new ArrayList<>()); + private static final List crossLanguageObjectsInstructionFiles = + Collections.synchronizedList(new ArrayList<>()); + + private static KeyPair RSA_KEY_PAIR_1; + + @BeforeAll + static void setupKeys() throws Exception { + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(2048); + RSA_KEY_PAIR_1 = keyPairGen.generateKeyPair(); + } + + /** + * Public accessors for decrypt tests to retrieve encrypted object keys and RSA key + */ + static List getCrossLanguageObjectsMetaDataMode() { + return new ArrayList<>(crossLanguageObjectsMetaDataMode); + } + + static List getCrossLanguageObjectsInstructionFiles() { + return new ArrayList<>(crossLanguageObjectsInstructionFiles); + } + + static KeyPair getRsaKeyPair() { + return RSA_KEY_PAIR_1; + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should encrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_allow_decrypt_should_encrypt_kc_gcm_kms( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), + crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should encrypt KC-GCM (instruction file)") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_encrypt_kc_gcm_ins_file_rsa( + TestUtils.LanguageServerTarget language + ) { + if (!RAW_SUPPORTED.contains(language.getLanguageName())) { + throw new TestAbortedException("Not encrypting raw keyring with: " + language.getLanguageName()); + } + + KeyMaterial rsaKey = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) + .build(); + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .keyMaterial(rsaKey).build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBaseInsFileMode + language.getLanguageName()), + crossLanguageObjectsInstructionFiles, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should encrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_encrypt_kc_gcm_kms( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), + crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with the default should encrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_the_default_should_encrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), + crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @AfterAll + static void signalEncryptionComplete() { + // Signal that all encryption tests have completed + encryptPhaseComplete.countDown(); + } + } + + /** + * KC-GCM Decryption Tests - Decrypt Phase + * + * These tests decrypt objects that were encrypted by EncryptTests. + * All tests in this class can run fully in parallel with each other. + * They depend on EncryptTests completing first (enforced by @Order). + */ + @Nested + @DisplayName("KC_GCMTestSuite - Decrypt") + class DecryptTests { + private static List crossLanguageObjectsMetaDataMode; + private static List crossLanguageObjectsInstructionFiles; + private static KeyPair RSA_KEY_PAIR_1; + private static final KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + @BeforeAll + static void setup() throws InterruptedException { + // Wait for all encryption tests to complete + encryptPhaseComplete.await(); + + // Import encrypted objects and RSA key from the encrypt phase + crossLanguageObjectsMetaDataMode = EncryptTests.getCrossLanguageObjectsMetaDataMode(); + crossLanguageObjectsInstructionFiles = EncryptTests.getCrossLanguageObjectsInstructionFiles(); + RSA_KEY_PAIR_1 = EncryptTests.getRsaKeyPair(); + + // Verify we have objects to decrypt + if (crossLanguageObjectsMetaDataMode.isEmpty()) { + throw new IllegalStateException( + "No encrypted objects found. Ensure EncryptTests runs first."); + } + } + + @ParameterizedTest(name = "{0}: Transition configured with the default should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_the_default_should_decrypt_kc_gcm_kms( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_allow_decrypt_should_decrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_decrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with the default should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_the_default_should_decrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should decrypt KC-GCM (instruction file)") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_decrypt_kc_gcm_ins_file( + final TestUtils.LanguageServerTarget language + ) { + if (!RAW_SUPPORTED.contains(language.getLanguageName())) { + throw new TestAbortedException("Not encrypting raw keyring with: " + language.getLanguageName()); + } + + KeyMaterial rsaKey = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) + .build(); + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .keyMaterial(rsaKey).build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsInstructionFiles, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Transition configured with default should decrypt KC-GCM (instruction file)") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_require_encrypt_require_decrypt_should_decrypt_kc_gcm_ins_file_rsa( + final TestUtils.LanguageServerTarget language + ) { + if (!RAW_SUPPORTED.contains(language.getLanguageName())) { + throw new TestAbortedException("Not encrypting raw keyring with: " + language.getLanguageName()); + } + + KeyMaterial rsaKey = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) + .build(); + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .keyMaterial(rsaKey).build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsInstructionFiles, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + } +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RangedGetTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RangedGetTests.java new file mode 100644 index 00000000..5a954e1f --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RangedGetTests.java @@ -0,0 +1,1687 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.encryption.s3; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static software.amazon.encryption.s3.TestUtils.*; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.CountDownLatch; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.amazonaws.services.s3.AmazonS3Encryption; +import com.amazonaws.services.s3.AmazonS3EncryptionClient; +import com.amazonaws.services.s3.model.CryptoConfiguration; +import com.amazonaws.services.s3.model.CryptoMode; +import com.amazonaws.services.s3.model.CryptoStorageMode; +import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; + +import software.amazon.encryption.s3.TestUtils.LanguageServerTarget; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.InstructionFileConfig; +import software.amazon.encryption.s3.model.GetObjectInput; +import software.amazon.encryption.s3.model.GetObjectOutput; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.S3ECConfig; + +/** + * Ranged Get Tests - S3 Encryption Client Cross-Language Compatibility + * + * PURPOSE: + * This test suite validates that ranged get operations (partial object reads) work correctly + * across all three encryption algorithms (CBC, GCM, KC-GCM) and that commitment validation + * occurs properly during ranged gets for KC-GCM encrypted objects. + * + * WHAT IS BEING TESTED: + * 1. Ranged gets successfully retrieve partial content from encrypted objects across all algorithms + * 2. Commitment validation is enforced during ranged gets for KC-GCM encrypted objects + * 3. Corrupted commitment metadata (removed, moved, or mutated) causes ranged gets to fail + * 4. Various byte ranges work correctly: start, end, middle, whole file, and auth tag only + * + * WHY THIS IS IMPORTANT: + * - Ranged gets are a critical S3 feature that must work with encrypted objects + * - KC-GCM's commitment mechanism must be validated even for partial reads to prevent + * commitment-based issues where an actor control the encryption keys + * - Cross-language compatibility ensures all SDKs handle ranged gets consistently + * - Edge cases (first/last bytes, auth tags) verify boundary condition handling + * + * TEST STRUCTURE: + * This suite uses a two-phase approach with enforced ordering: + * 1. EncryptTests - Encrypts objects with CBC, GCM, and KC-GCM algorithms + * - Creates corrupted KC-GCM test cases with manipulated commitment metadata + * - All encrypt tests can run in parallel within this phase + * 2. RangedGetTests - Waits for encryption to complete, then tests ranged gets + * - Tests successful ranged gets on valid objects + * - Tests failed ranged gets on corrupted commitment objects + * - All ranged get tests can run in parallel within this phase + * + * Coordination uses a CountDownLatch to ensure all encryption completes before ranged gets begin. + * + * INPUT DIMENSIONS: + * - Encryption Algorithm: CBC, GCM, KC-GCM + * - Language Implementation: All languages supporting RANGED_GETS_SUPPORTED + * - Byte Range Types: + * * Start (bytes 0-99) + * * End (last 100 bytes) + * * Middle (100 bytes centered in file) + * * Whole file (all bytes) + * * Auth tag only (last 16 bytes for authenticated algorithms) + * - Storage Mode (KC-GCM only): + * * Object Metadata Storage (all metadata in object, no instruction file) + * * Instruction File Storage (c/d/i in metadata, x-amz-3/w/m/t in instruction file) + * - Commitment State (KC-GCM only): + * * Valid - Object Metadata Storage (original and good-copy) + * * Valid - Instruction File Storage (original and good-copy) + * * Corrupted - Object Metadata Storage: + * - Mutated c/d/i: bit flipped in metadata values + * - Invalid c length: c < 28 bytes in metadata + * - Invalid c length: c > 28 bytes in metadata + * * Corrupted - Instruction File Storage: + * - Commitment duplicated: c/d/i in instruction file (already in metadata) + * - Commitment removed: c/d/i removed from metadata + * - Mutated c/d/i in metadata: bit flipped + * - Mutated c/d/i in instruction file: bit flipped + * - Invalid c length: c < 28 bytes in metadata + * - Invalid c length: c > 28 bytes in metadata + * + * EXPECTED RESULTS: + * - Positive: Ranged gets on valid CBC, GCM, KC-GCM objects return correct partial content + * - Negative: Ranged gets on corrupted KC-GCM objects fail with commitment validation errors + * + * REPRESENTATIVE VALUES: + * - Bit flip position: Randomly selected per test run, included in object key name + * - File size: Object keys themselves (short strings) serve as representative small files + * - Byte ranges: Fixed patterns covering important boundary conditions + * + * SCOPE: + * - Languages in RANGED_GETS_SUPPORTED set are tested, + * the encrypt tests are to create values that are then tested. + * - CBC and GCM tests validate ranged get functionality works + * - KC-GCM tests focus on commitment validation during ranged gets + */ +public class RangedGetTests { + // Synchronization latch - released when encrypt phase completes + private static final CountDownLatch encryptPhaseComplete = new CountDownLatch(1); + + // Random number generator for bit flipping (seeded for reproducibility) + private static final Random random = new Random(System.currentTimeMillis()); + + // Object key suffixes for test copies + private static final String SUFFIX_GOOD_COPY = "-good-copy"; + private static final String SUFFIX_BAD_MUTATED_C = "-bad-mutated-c-bit-"; + private static final String SUFFIX_BAD_MUTATED_D = "-bad-mutated-d-bit-"; + private static final String SUFFIX_BAD_MUTATED_I = "-bad-mutated-i-bit-"; + private static final String SUFFIX_BAD_INVALID_D_LENGTH_SHORT = "-bad-invalid-d-length-short"; + private static final String SUFFIX_BAD_INVALID_D_LENGTH_LONG = "-bad-invalid-d-length-long"; + private static final String SUFFIX_BAD_COMMITMENT_IN_INSTRUCTION = "-bad-commitment-in-instruction"; + + /** + * Encryption Tests - Encrypt Phase + * + * These tests encrypt objects using CBC, GCM, and KC-GCM algorithms, then create + * corrupted copies for failure testing. All tests in this class can run in parallel. + */ + @Nested + @DisplayName("RangedGetTests - Encrypt") + class EncryptTests { + private static final String sharedObjectKeyBase = "test-ranged-get"; + private static KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + // Thread-safe lists for storing encrypted object keys + private static final List cbcObjects = + Collections.synchronizedList(new ArrayList<>()); + private static final List gcmObjects = + Collections.synchronizedList(new ArrayList<>()); + // KC-GCM with Object Metadata Storage (all metadata in object) + private static final List kcGcmObjectsMetadata = + Collections.synchronizedList(new ArrayList<>()); + // KC-GCM with Instruction File Storage (c/d/i in metadata, rest in instruction file) + private static final List kcGcmObjectsInstruction = + Collections.synchronizedList(new ArrayList<>()); + // Corruption test lists for metadata storage mode + private static final List mutatedCObjectsMetadata = + Collections.synchronizedList(new ArrayList<>()); + private static final List mutatedDObjectsMetadata = + Collections.synchronizedList(new ArrayList<>()); + private static final List mutatedIObjectsMetadata = + Collections.synchronizedList(new ArrayList<>()); + private static final List invalidDLengthShortMetadata = + Collections.synchronizedList(new ArrayList<>()); + private static final List invalidDLengthLongMetadata = + Collections.synchronizedList(new ArrayList<>()); + // Corruption test lists for instruction file storage mode + private static final List mutatedCObjectsInstruction = + Collections.synchronizedList(new ArrayList<>()); + private static final List mutatedDObjectsInstruction = + Collections.synchronizedList(new ArrayList<>()); + private static final List mutatedIObjectsInstruction = + Collections.synchronizedList(new ArrayList<>()); + private static final List invalidDLengthShortInstruction = + Collections.synchronizedList(new ArrayList<>()); + private static final List invalidDLengthLongInstruction = + Collections.synchronizedList(new ArrayList<>()); + + private static KeyMaterial RSA_KEY; + private static KeyMaterial AES_KEY; + + @BeforeAll + static void setupKeys() throws Exception { + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(2048); + KeyPair keyPair = keyPairGen.generateKeyPair(); + + RSA_KEY = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(keyPair.getPrivate().getEncoded())) + .build(); + + KeyGenerator keyGen = KeyGenerator.getInstance("AES"); + keyGen.init(256); + SecretKey aesSecretKey = keyGen.generateKey(); + + AES_KEY = KeyMaterial.builder() + .aesKey(ByteBuffer.wrap(aesSecretKey.getEncoded())) + .build(); + } + + /** + * Public accessors for ranged get tests to retrieve encrypted object keys + */ + static List getCbcObjects() { + return new ArrayList<>(cbcObjects); + } + + static List getGcmObjects() { + return new ArrayList<>(gcmObjects); + } + + static List getKcGcmObjectsMetadata() { + return new ArrayList<>(kcGcmObjectsMetadata); + } + + static List getKcGcmObjectsInstruction() { + return new ArrayList<>(kcGcmObjectsInstruction); + } + + static List getMutatedCObjectsMetadata() { + return new ArrayList<>(mutatedCObjectsMetadata); + } + + static List getMutatedDObjectsMetadata() { + return new ArrayList<>(mutatedDObjectsMetadata); + } + + static List getMutatedIObjectsMetadata() { + return new ArrayList<>(mutatedIObjectsMetadata); + } + + static List getInvalidDLengthShortMetadata() { + return new ArrayList<>(invalidDLengthShortMetadata); + } + + static List getInvalidDLengthLongMetadata() { + return new ArrayList<>(invalidDLengthLongMetadata); + } + + static List getMutatedCObjectsInstruction() { + return new ArrayList<>(mutatedCObjectsInstruction); + } + + static List getMutatedDObjectsInstruction() { + return new ArrayList<>(mutatedDObjectsInstruction); + } + + static List getMutatedIObjectsInstruction() { + return new ArrayList<>(mutatedIObjectsInstruction); + } + + static List getInvalidDLengthShortInstruction() { + return new ArrayList<>(invalidDLengthShortInstruction); + } + + static List getInvalidDLengthLongInstruction() { + return new ArrayList<>(invalidDLengthLongInstruction); + } + + static KeyMaterial getKmsKeyArn() { + return kmsKeyArn; + } + + static KeyMaterial getRsaKey() { + return RSA_KEY; + } + + static KeyMaterial getAesKey() { + return AES_KEY; + } + + // GCM can be encrypted by transition and improved clients + public static Stream transitionAndImprovedForGCM() { + return Stream.concat( + transitionClientsForTest(), + improvedClientsForTest() + ); + } + + // KC-GCM can be encrypted by improved clients only + public static Stream improvedClientsForKCGCM() { + return improvedClientsForTest(); + } + + public static Stream improvedClientsCanPutKMSWithInstructionFile() { + return improvedClientsForTest() + .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + + @org.junit.jupiter.api.Test + void encryptCbcForRangedGets() { + // Use old V1 client for CBC encryption (legacy algorithm) + // Only Java V1 client is available - no V1 test servers for other languages + EncryptionMaterialsProvider materialsProvider = new KMSEncryptionMaterialsProvider(TestUtils.KMS_KEY_ARN); + + CryptoConfiguration v1Config = + new CryptoConfiguration(CryptoMode.EncryptionOnly) + .withStorageMode(CryptoStorageMode.ObjectMetadata) + .withAwsKmsRegion(TestUtils.KMS_REGION); + + AmazonS3Encryption v1Client = AmazonS3EncryptionClient.encryptionBuilder() + .withCryptoConfiguration(v1Config) + .withEncryptionMaterials(materialsProvider) + .build(); + + String objectKey = appendTestSuffix(sharedObjectKeyBase + "-cbc-java"); + v1Client.putObject(TestUtils.BUCKET, objectKey, objectKey); + cbcObjects.add(objectKey); + } + + @ParameterizedTest(name = "{0}: Encrypt GCM for ranged get testing") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$EncryptTests#transitionAndImprovedForGCM") + void encryptGcmForRangedGets(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBase + "-gcm-" + language.getLanguageName()), + gcmObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + ); + } + + @ParameterizedTest(name = "{0}: Encrypt KC-GCM with Object Metadata Storage for ranged get testing") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$EncryptTests#improvedClientsForKCGCM") + void encryptKcGcmMetadataForRangedGets(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBase + "-kc-gcm-metadata-" + language.getLanguageName()), + kcGcmObjectsMetadata, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + + @ParameterizedTest(name = "{0}: Encrypt KC-GCM with Instruction file Storage for ranged get testing") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$EncryptTests#improvedClientsCanPutKMSWithInstructionFile") + void encryptKcGcmInstructionFileForRangedGets(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .instructionFileConfig( + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() + ) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBase + "-kc-gcm-instruction-java" + language.getLanguageName()), + kcGcmObjectsInstruction, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + /** + * Flips a random bit in the given byte array + * @param data The byte array to modify + * @return The bit position that was flipped + */ + static int flipRandomBit(byte[] data) { + if (data.length == 0) { + return -1; + } + int bitPosition = random.nextInt(data.length * 8); + int byteIndex = bitPosition / 8; + int bitIndex = bitPosition % 8; + data[byteIndex] ^= (1 << bitIndex); + return bitPosition; + } + + /** + * Creates corrupted copies of KC-GCM objects for failure testing + * Handles both object metadata storage and instruction file storage modes + */ + static void createCorruptedCopies() throws Exception { + try (S3Client ptS3Client = S3Client.create()) { + ObjectMapper mapper = new ObjectMapper(); + + // Process metadata storage mode objects (all V3 keys in metadata, no instruction file) + for (String objectKey : kcGcmObjectsMetadata) { + ResponseBytes encryptedObject = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + + byte[] objectData = encryptedObject.asByteArray(); + Map objectMetadata = encryptedObject.response().metadata(); + + // Create good copy + putObjectWithMetadata(ptS3Client, objectKey + SUFFIX_GOOD_COPY, objectData, objectMetadata); + + // Extract commitment values from metadata + String commitC = objectMetadata.get("x-amz-c"); + String commitD = objectMetadata.get("x-amz-d"); + String commitI = objectMetadata.get("x-amz-i"); + + // Create mutated commitment copies in metadata + if (commitC != null) { + byte[] commitCBytes = Base64.getDecoder().decode(commitC); + int bitPos = flipRandomBit(commitCBytes); + String mutatedC = Base64.getEncoder().encodeToString(commitCBytes); + Map mutatedMetadata = new java.util.HashMap<>(objectMetadata); + mutatedMetadata.put("x-amz-c", mutatedC); + String mutatedKey = objectKey + SUFFIX_BAD_MUTATED_C + bitPos; + putObjectWithMetadata(ptS3Client, mutatedKey, objectData, mutatedMetadata); + mutatedCObjectsMetadata.add(mutatedKey); + } + + if (commitD != null) { + byte[] commitDBytes = Base64.getDecoder().decode(commitD); + int bitPos = flipRandomBit(commitDBytes); + String mutatedD = Base64.getEncoder().encodeToString(commitDBytes); + Map mutatedMetadata = new java.util.HashMap<>(objectMetadata); + mutatedMetadata.put("x-amz-d", mutatedD); + String mutatedKey = objectKey + SUFFIX_BAD_MUTATED_D + bitPos; + putObjectWithMetadata(ptS3Client, mutatedKey, objectData, mutatedMetadata); + mutatedDObjectsMetadata.add(mutatedKey); + } + + if (commitI != null) { + byte[] commitIBytes = Base64.getDecoder().decode(commitI); + int bitPos = flipRandomBit(commitIBytes); + String mutatedI = Base64.getEncoder().encodeToString(commitIBytes); + Map mutatedMetadata = new java.util.HashMap<>(objectMetadata); + mutatedMetadata.put("x-amz-i", mutatedI); + String mutatedKey = objectKey + SUFFIX_BAD_MUTATED_I + bitPos; + putObjectWithMetadata(ptS3Client, mutatedKey, objectData, mutatedMetadata); + mutatedIObjectsMetadata.add(mutatedKey); + } + + // Create invalid D length copies (metadata storage) + if (commitD != null) { + byte[] commitDBytes = Base64.getDecoder().decode(commitD); + + // Short D (< 28 bytes) - truncate to 20 bytes + int shortLength = Math.min(20, commitDBytes.length); + byte[] shortDBytes = new byte[shortLength]; + System.arraycopy(commitDBytes, 0, shortDBytes, 0, shortLength); + String shortD = Base64.getEncoder().encodeToString(shortDBytes); + Map shortDMetadata = new java.util.HashMap<>(objectMetadata); + shortDMetadata.put("x-amz-d", shortD); + String shortDKey = objectKey + SUFFIX_BAD_INVALID_D_LENGTH_SHORT; + putObjectWithMetadata(ptS3Client, shortDKey, objectData, shortDMetadata); + invalidDLengthShortMetadata.add(shortDKey); + + // Long D (> 28 bytes) - extend to 40 bytes + byte[] longDBytes = new byte[40]; + System.arraycopy(commitDBytes, 0, longDBytes, 0, commitDBytes.length); + // Fill remaining bytes with zeros + for (int i = commitDBytes.length; i < 40; i++) { + longDBytes[i] = 0; + } + String longD = Base64.getEncoder().encodeToString(longDBytes); + Map longDMetadata = new java.util.HashMap<>(objectMetadata); + longDMetadata.put("x-amz-d", longD); + String longDKey = objectKey + SUFFIX_BAD_INVALID_D_LENGTH_LONG; + putObjectWithMetadata(ptS3Client, longDKey, objectData, longDMetadata); + invalidDLengthLongMetadata.add(longDKey); + } + } + + // Process instruction file storage mode objects (c/d/i in metadata, x-amz-3/w/m/t in instruction file) + for (String objectKey : kcGcmObjectsInstruction) { + // Get the encrypted object + ResponseBytes encryptedObject = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + + byte[] objectData = encryptedObject.asByteArray(); + Map objectMetadata = encryptedObject.response().metadata(); + + // Get the instruction file + ResponseBytes instructionObject = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(TestUtils.BUCKET) + .key(objectKey + ".instruction") + .build()); + + String originalInstructionFileJson = new String(instructionObject.asByteArray(), StandardCharsets.UTF_8); + + // Create good copy (both object and instruction file) + putObjectWithInstructionFile( + ptS3Client, + objectKey + SUFFIX_GOOD_COPY, + objectData, + objectMetadata, + originalInstructionFileJson + ); + + // Extract commitment values from metadata + String commitC = objectMetadata.get("x-amz-c"); + String commitD = objectMetadata.get("x-amz-d"); + String commitI = objectMetadata.get("x-amz-i"); + + // Corruption: Add c/d/i to instruction file (duplication - should fail) + Map corruptedInstructionMap = mapper.readValue(originalInstructionFileJson, Map.class); + corruptedInstructionMap.put("x-amz-c", commitC); + corruptedInstructionMap.put("x-amz-d", commitD); + corruptedInstructionMap.put("x-amz-i", commitI); + String corruptedInstructionJson = mapper.writeValueAsString(corruptedInstructionMap); + + putObjectWithInstructionFile( + ptS3Client, + objectKey + SUFFIX_BAD_COMMITMENT_IN_INSTRUCTION, + objectData, + objectMetadata, + corruptedInstructionJson + ); + + // Create mutated commitment copies in metadata + if (commitC != null) { + byte[] commitCBytes = Base64.getDecoder().decode(commitC); + int bitPos = flipRandomBit(commitCBytes); + String mutatedC = Base64.getEncoder().encodeToString(commitCBytes); + Map mutatedMetadata = new java.util.HashMap<>(objectMetadata); + mutatedMetadata.put("x-amz-c", mutatedC); + String mutatedKey = objectKey + SUFFIX_BAD_MUTATED_C + bitPos; + putObjectWithInstructionFile(ptS3Client, mutatedKey, objectData, mutatedMetadata, originalInstructionFileJson); + mutatedCObjectsInstruction.add(mutatedKey); + } + + if (commitD != null) { + byte[] commitDBytes = Base64.getDecoder().decode(commitD); + int bitPos = flipRandomBit(commitDBytes); + String mutatedD = Base64.getEncoder().encodeToString(commitDBytes); + Map mutatedMetadata = new java.util.HashMap<>(objectMetadata); + mutatedMetadata.put("x-amz-d", mutatedD); + String mutatedKey = objectKey + SUFFIX_BAD_MUTATED_D + bitPos; + putObjectWithInstructionFile(ptS3Client, mutatedKey, objectData, mutatedMetadata, originalInstructionFileJson); + mutatedDObjectsInstruction.add(mutatedKey); + } + + if (commitI != null) { + byte[] commitIBytes = Base64.getDecoder().decode(commitI); + int bitPos = flipRandomBit(commitIBytes); + String mutatedI = Base64.getEncoder().encodeToString(commitIBytes); + Map mutatedMetadata = new java.util.HashMap<>(objectMetadata); + mutatedMetadata.put("x-amz-i", mutatedI); + String mutatedKey = objectKey + SUFFIX_BAD_MUTATED_I + bitPos; + putObjectWithInstructionFile(ptS3Client, mutatedKey, objectData, mutatedMetadata, originalInstructionFileJson); + mutatedIObjectsInstruction.add(mutatedKey); + } + + // Create invalid D length copies (instruction file storage) + if (commitD != null) { + byte[] commitDBytes = Base64.getDecoder().decode(commitD); + + // Short D (< 28 bytes) - truncate to 20 bytes + int shortLength = Math.min(20, commitDBytes.length); + byte[] shortDBytes = new byte[shortLength]; + System.arraycopy(commitDBytes, 0, shortDBytes, 0, shortLength); + String shortD = Base64.getEncoder().encodeToString(shortDBytes); + Map shortDMetadata = new java.util.HashMap<>(objectMetadata); + shortDMetadata.put("x-amz-d", shortD); + String shortDKey = objectKey + SUFFIX_BAD_INVALID_D_LENGTH_SHORT; + putObjectWithInstructionFile(ptS3Client, shortDKey, objectData, shortDMetadata, originalInstructionFileJson); + invalidDLengthShortInstruction.add(shortDKey); + + // Long D (> 28 bytes) - extend to 40 bytes + byte[] longDBytes = new byte[40]; + System.arraycopy(commitDBytes, 0, longDBytes, 0, commitDBytes.length); + // Fill remaining bytes with zeros + for (int i = commitDBytes.length; i < 40; i++) { + longDBytes[i] = 0; + } + String longD = Base64.getEncoder().encodeToString(longDBytes); + Map longDMetadata = new java.util.HashMap<>(objectMetadata); + longDMetadata.put("x-amz-d", longD); + String longDKey = objectKey + SUFFIX_BAD_INVALID_D_LENGTH_LONG; + putObjectWithInstructionFile(ptS3Client, longDKey, objectData, longDMetadata, originalInstructionFileJson); + invalidDLengthLongInstruction.add(longDKey); + } + } + } + } + + static void putObjectWithMetadata( + S3Client ptS3Client, + String objectKey, + byte[] objectData, + Map objectMetadata + ) { + ptS3Client.putObject(builder -> builder + .bucket(TestUtils.BUCKET) + .key(objectKey) + .metadata(objectMetadata) + .build(), + software.amazon.awssdk.core.sync.RequestBody.fromBytes(objectData)); + } + + static void putObjectWithInstructionFile( + S3Client ptS3Client, + String objectKey, + byte[] objectData, + Map objectMetadata, + String instructionFileJson + ) { + // Put the encrypted object + ptS3Client.putObject(builder -> builder + .bucket(TestUtils.BUCKET) + .key(objectKey) + .metadata(objectMetadata) + .build(), + software.amazon.awssdk.core.sync.RequestBody.fromBytes(objectData)); + + // Put the instruction file + ptS3Client.putObject(builder -> builder + .bucket(TestUtils.BUCKET) + .key(objectKey + ".instruction") + .build(), + software.amazon.awssdk.core.sync.RequestBody.fromBytes( + instructionFileJson.getBytes(StandardCharsets.UTF_8))); + } + + @AfterAll + static void signalEncryptionComplete() throws Exception { + createCorruptedCopies(); + + // Signal that all encryption tests have completed + encryptPhaseComplete.countDown(); + } + } + + /** + * Ranged Get Tests - Test Phase + * + * These tests perform ranged get operations on objects encrypted by EncryptTests. + * All tests in this class can run fully in parallel with each other. + * They depend on EncryptTests completing first. + */ + @Nested + @DisplayName("RangedGetTests - RangedGet") + class RangedGetTestsNested { + private static List cbcObjects; + private static List gcmObjects; + private static List kcGcmObjects; + private static List kcGcmObjectsInstruction; + private static List mutatedCObjects; + private static List mutatedDObjects; + private static List mutatedIObjects; + private static List mutatedCObjectsInstruction; + private static List mutatedDObjectsInstruction; + private static List mutatedIObjectsInstruction; + private static KeyMaterial kmsKeyArn; + private static KeyMaterial RSA_KEY; + private static KeyMaterial AES_KEY; + + @BeforeAll + static void setup() throws InterruptedException { + // Wait for all encryption tests to complete + encryptPhaseComplete.await(); + + // Import encrypted objects from the encrypt phase + cbcObjects = EncryptTests.getCbcObjects(); + gcmObjects = EncryptTests.getGcmObjects(); + // Import KC-GCM objects for both storage modes + kcGcmObjects = EncryptTests.getKcGcmObjectsMetadata(); + kcGcmObjectsInstruction = EncryptTests.getKcGcmObjectsInstruction(); + // Import corrupted objects for metadata storage mode + mutatedCObjects = EncryptTests.getMutatedCObjectsMetadata(); + mutatedDObjects = EncryptTests.getMutatedDObjectsMetadata(); + mutatedIObjects = EncryptTests.getMutatedIObjectsMetadata(); + // Import corrupted objects for instruction file storage mode + mutatedCObjectsInstruction = EncryptTests.getMutatedCObjectsInstruction(); + mutatedDObjectsInstruction = EncryptTests.getMutatedDObjectsInstruction(); + mutatedIObjectsInstruction = EncryptTests.getMutatedIObjectsInstruction(); + kmsKeyArn = EncryptTests.getKmsKeyArn(); + RSA_KEY = EncryptTests.getRsaKey(); + AES_KEY = EncryptTests.getAesKey(); + + // Verify we have objects to test + if (cbcObjects.isEmpty() && gcmObjects.isEmpty() && kcGcmObjects.isEmpty()) { + throw new IllegalStateException( + "No encrypted objects found. Ensure EncryptTests runs first."); + } + } + + public static Stream rangedGetSupportedClients() { + Stream improved = improvedClientsForTest() + .filter(target -> RANGED_GETS_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + Stream transition = transitionClientsForTest() + .filter(target -> RANGED_GETS_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + return Stream.concat(improved, transition); + } + + public static Stream rangedGetCBCSupportedClients() { + return rangedGetSupportedClients() + // This is just a quick hack. Perhaps it would be good to have an equivalent group for languages. + .filter(target -> !((LanguageServerTarget) target.get()[0]).getLanguageName().startsWith("CPP")); + } + + // CBC Ranged Get Tests + + @ParameterizedTest(name = "{0}: Successfully ranged get CBC objects - start range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetCBCSupportedClients") + void rangedGetCbcStartSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .enableLegacyWrappingAlgorithms(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + cbcObjects, + 0, + 5, + EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get CBC objects - end range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetCBCSupportedClients") + void rangedGetCbcEndSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .enableLegacyWrappingAlgorithms(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // For each object, get its length and test the last 5 bytes + for (String objectKey : cbcObjects) { + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + long rangeStart = Math.max(0, objectLength - 5); + long rangeEnd = objectLength - 1; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(objectKey), + rangeStart, + rangeEnd, + EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF + ); + } + } + + @ParameterizedTest(name = "{0}: Successfully ranged get CBC objects - middle range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetCBCSupportedClients") + void rangedGetCbcMiddleSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .enableLegacyWrappingAlgorithms(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + cbcObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get CBC objects - whole file") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetCBCSupportedClients") + void rangedGetCbcWholeFileSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .enableLegacyWrappingAlgorithms(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // For each object, get its length and test the whole file using range + for (String objectKey : cbcObjects) { + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(objectKey), + 0, + objectLength - 1, + EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF + ); + } + } + + // // GCM Ranged Get Tests + + @ParameterizedTest(name = "{0}: Successfully ranged get GCM objects - start range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetGcmStartSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + gcmObjects, + 0, + 5, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get GCM objects - end range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetGcmEndSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // For each object, get its length and test the last 5 bytes + for (String objectKey : gcmObjects) { + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + long rangeStart = Math.max(0, objectLength - 5); + long rangeEnd = objectLength - 1; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(objectKey), + rangeStart, + rangeEnd, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + ); + } + } + + @ParameterizedTest(name = "{0}: Successfully ranged get GCM objects - middle range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetGcmMiddleSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + gcmObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get GCM objects - whole file") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetGcmWholeFileSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // For each object, get its length and test the whole file using range + for (String objectKey : gcmObjects) { + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(objectKey), + 0, + objectLength - 1, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + ); + } + } + + @ParameterizedTest(name = "{0}: Successfully ranged get GCM objects - Include tag") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetGcmTagOnlySucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + gcmObjects, + 10, + 1000, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + ); + } + + // KC-GCM Ranged Get Tests - Valid Objects + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM objects - start range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmStartSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjects, + 0, + 5, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjects + .stream() + .map(key -> key + "-good-copy") + .collect(Collectors.toList()), + 0, + 5, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM objects - middle range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmMiddleSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjects + .stream() + .map(key -> key + "-good-copy") + .collect(Collectors.toList()), + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM objects - Include tag") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmTagOnlySucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjects, + 10, + 1000, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjects + .stream() + .map(key -> key + "-good-copy") + .collect(Collectors.toList()), + 10, + 1000, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM objects - end range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmEndSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // Test original objects + for (String objectKey : kcGcmObjects) { + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + long rangeStart = Math.max(0, objectLength - 5); + long rangeEnd = objectLength - 1; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(objectKey), + rangeStart, + rangeEnd, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + // Test good-copy objects + for (String objectKey : kcGcmObjects) { + String goodCopyKey = objectKey + "-good-copy"; + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(goodCopyKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + long rangeStart = Math.max(0, objectLength - 5); + long rangeEnd = objectLength - 1; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(goodCopyKey), + rangeStart, + rangeEnd, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + } + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM objects - whole file") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmWholeFileSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // Test original objects + for (String objectKey : kcGcmObjects) { + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(objectKey), + 0, + objectLength - 1, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + // Test good-copy objects + for (String objectKey : kcGcmObjects) { + String goodCopyKey = objectKey + "-good-copy"; + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(goodCopyKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(goodCopyKey), + 0, + objectLength - 1, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + } + + // KC-GCM Instruction File Storage - Valid Object Tests + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM Instruction File objects - start range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionStartSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjectsInstruction, + 0, + 5, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjectsInstruction + .stream() + .map(key -> key + "-good-copy") + .collect(Collectors.toList()), + 0, + 5, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM Instruction File objects - middle range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionMiddleSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjectsInstruction, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjectsInstruction + .stream() + .map(key -> key + "-good-copy") + .collect(Collectors.toList()), + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM Instruction File objects - Include tag") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionTagOnlySucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjectsInstruction, + 10, + 1000, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjectsInstruction + .stream() + .map(key -> key + "-good-copy") + .collect(Collectors.toList()), + 10, + 1000, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM Instruction File objects - end range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionEndSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // Test original objects + for (String objectKey : kcGcmObjectsInstruction) { + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + long rangeStart = Math.max(0, objectLength - 5); + long rangeEnd = objectLength - 1; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(objectKey), + rangeStart, + rangeEnd, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + // Test good-copy objects + for (String objectKey : kcGcmObjectsInstruction) { + String goodCopyKey = objectKey + "-good-copy"; + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(goodCopyKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + long rangeStart = Math.max(0, objectLength - 5); + long rangeEnd = objectLength - 1; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(goodCopyKey), + rangeStart, + rangeEnd, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + } + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM Instruction File objects - whole file") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionWholeFileSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // Test original objects + for (String objectKey : kcGcmObjectsInstruction) { + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(objectKey), + 0, + objectLength - 1, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + // Test good-copy objects + for (String objectKey : kcGcmObjectsInstruction) { + String goodCopyKey = objectKey + "-good-copy"; + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(goodCopyKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(goodCopyKey), + 0, + objectLength - 1, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + } + + // KC-GCM Ranged Get Tests - Failure Cases + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM Instruction File with commitment duplicated in instruction file") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionCommitmentInInstructionFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // Test instruction file storage mode objects with c/d/i duplicated into instruction file + TestUtils.RangedGet_fails( + client, + S3ECId, + kcGcmObjectsInstruction.stream() + .map(key -> key + "-bad-commitment-in-instruction") + .collect(Collectors.toList()), + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM with mutated commitment C") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmMutatedCFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + mutatedCObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM with mutated commitment D") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmMutatedDFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + mutatedDObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM with mutated commitment I") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmMutatedIFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + mutatedIObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM Instruction File with mutated commitment C in metadata") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionMutatedCFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + mutatedCObjectsInstruction, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM Instruction File with mutated commitment D in metadata") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionMutatedDFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + mutatedDObjectsInstruction, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM Instruction File with mutated commitment I in metadata") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionMutatedIFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + mutatedIObjectsInstruction, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM with invalid C length (too short) in metadata") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmMetadataInvalidCLengthShortFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + List invalidDLengthShortObjects = EncryptTests.getInvalidDLengthShortMetadata(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + invalidDLengthShortObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM with invalid D length (too long) in metadata") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmMetadataInvalidDLengthLongFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + List invalidDLengthLongObjects = EncryptTests.getInvalidDLengthLongMetadata(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + invalidDLengthLongObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM Instruction File with invalid D length (too short) in metadata") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionInvalidDLengthShortFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + List invalidDLengthShortObjects = EncryptTests.getInvalidDLengthShortInstruction(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + invalidDLengthShortObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM Instruction File with invalid D length (too long) in metadata") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionInvalidDLengthLongFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + List invalidDLengthLongObjects = EncryptTests.getInvalidDLengthLongInstruction(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + invalidDLengthLongObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + } +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ReEncryptTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ReEncryptTests.java new file mode 100644 index 00000000..3053afb6 --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ReEncryptTests.java @@ -0,0 +1,648 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.encryption.s3; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static software.amazon.encryption.s3.TestUtils.*; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.stream.Stream; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import com.amazonaws.services.s3.AmazonS3Encryption; +import com.amazonaws.services.s3.AmazonS3EncryptionClient; +import com.amazonaws.services.s3.model.CryptoConfiguration; +import com.amazonaws.services.s3.model.CryptoMode; +import com.amazonaws.services.s3.model.CryptoStorageMode; +import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; + +import software.amazon.encryption.s3.TestUtils.LanguageServerTarget; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.InstructionFileConfig; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.ReEncryptInput; +import software.amazon.encryption.s3.model.ReEncryptOutput; +import software.amazon.encryption.s3.model.S3ECConfig; +import software.amazon.encryption.s3.model.S3EncryptionClientError; + +/** + * ReEncrypt Instruction File Tests - S3 Encryption Client Cross-Language Compatibility + * + * PURPOSE: + * This test suite validates that instruction file re-encryption enables key rotation without + * re-uploading encrypted objects, and that re-encrypted objects maintain cross-language + * compatibility and commitment validation guarantees. + * + * WHAT IS BEING TESTED: + * 1. Instruction file re-encryption for KC-GCM algorithm with raw keyrings + * 2. Re-encryption across different raw keyring types (AES, RSA) + * 3. Same-type keyring rotation (AES => AES, RSA => RSA) + * 4. Cross-type keyring rotation (AES => RSA, RSA => AES) + * 5. Default instruction file suffix (.instruction) and custom suffixes (.instruction-rsa, .instruction-aes) + * 6. Cross-language compatibility: all languages can decrypt after re-encryption + * 7. Rotation enforcement to prevent re-encryption with the same key + * + * WHY THIS IS IMPORTANT: + * - Key rotation is a critical security operation that should not require expensive object re-uploads + * - ReEncryptInstructionFile enables updating the encrypted data key without touching the ciphertext + * - Raw keyrings (AES, RSA) provide direct key material access required for re-encryption + * - Cross-type rotation (e.g., AES to RSA) enables flexibility in key management strategies + * - Commitment validation must be maintained even when instruction files are re-encrypted + * - Cross-language compatibility ensures key rotation doesn't break existing clients + * - Rotation enforcement prevents accidental re-encryption with the same key material + * - Custom instruction file suffixes enable sharing encrypted objects with partners + * + * TEST STRUCTURE: + * This suite uses a three-phase approach with enforced ordering: + * 1. EncryptTests - Encrypts objects with instruction files using AES and RSA keyrings + * - All encrypt tests can run in parallel within this phase + * - Signals encryptPhaseComplete latch when done + * 2. ReEncryptTests - Waits for encryption to complete, then re-encrypts instruction files + * - Tests same-type rotations (AES => AES, RSA => RSA) + * - Tests cross-type rotations (AES => RSA with .instruction-rsa suffix, RSA => AES with .instruction-aes suffix) + * - Tests rotation enforcement (same key rejection) + * - All re-encrypt tests can run in parallel within this phase + * - Tracks which objects were re-encrypted to which keys to prevent conflicts + * - Signals reEncryptPhaseComplete latch when done + * 3. DecryptReEncryptedTests - Waits for re-encryption to complete, then tests decryption + * - Tests cross-language decryption compatibility after re-encryption + * - Uses tracked object lists to decrypt with correct keys and custom instruction file suffixes + * - All decrypt tests can run in parallel within this phase + * + * Coordination uses two CountDownLatches: + * - encryptPhaseComplete: Ensures all encryption completes before re-encryption begins + * - reEncryptPhaseComplete: Ensures all re-encryption completes before decryption begins + * + * INPUT DIMENSIONS: + * - Source Key Material: AES (256-bit), RSA (2048-bit key pairs) + * - Destination Key Material: Different AES or RSA keys (raw keyrings) + * - Encryption Algorithm: KC-GCM (ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + * - Instruction File Suffix: default (.instruction), custom (.instruction-rsa, .instruction-aes) + * - Language for Re-encryption: Java V3-Transition, Java V4 (RE_ENCRYPT_SUPPORTED) + * - Language for Decryption: All languages supporting instruction files + * - Rotation Enforcement: enforceRotation flag (true/false) + * + * EXPECTED RESULTS: + * - Positive: Re-encryption succeeds with different key material, all languages can decrypt + * - Negative: Re-encryption fails when enforceRotation detects same key material + * + * REPRESENTATIVE VALUES: + * - Object keys themselves (short strings) serve as representative small plaintext files + * - Instruction file suffix: ".instruction" (default), ".instruction-rsa", ".instruction-aes" + * - Key materials: Generated once per type and reused across tests + * + * FILTERING: + * - Only languages in RE_ENCRYPT_SUPPORTED can perform re-encryption operations + * - Languages in INSTRUCTION_FILE_GET_UNSUPPORTED cannot decrypt with instruction files + * + * NOTE: KMS keyrings are NOT supported for re-encryption as the reEncryptInstructionFile + * method requires RawKeyring instances (AES or RSA) which provide direct access to key material. + * + */ +public class ReEncryptTests { + // Synchronization latches for three-phase coordination + private static final CountDownLatch encryptPhaseComplete = new CountDownLatch(1); + private static final CountDownLatch reEncryptPhaseComplete = new CountDownLatch(1); + + // Tracking lists for re-encrypted objects - shared across nested test classes + private static final List reEncryptedAesToAes = Collections.synchronizedList(new ArrayList<>()); + private static final List reEncryptedRsaToRsa = Collections.synchronizedList(new ArrayList<>()); + private static final List reEncryptedAesToRsa = Collections.synchronizedList(new ArrayList<>()); + private static final List reEncryptedRsaToAesDefault = Collections.synchronizedList(new ArrayList<>()); + private static final List reEncryptedAesToRsaDefault = Collections.synchronizedList(new ArrayList<>()); + + @Nested + @DisplayName("ReEncryptTests - Encrypt") + class EncryptTests { + private static final String sharedObjectKeyBase = "test-reencrypt"; + + private static SecretKey aesKey1, aesKey2; + private static KeyMaterial aesKeyMaterial1, aesKeyMaterial2; + private static KeyPair rsaKeyPair1, rsaKeyPair2; + private static KeyMaterial rsaKeyMaterial1, rsaKeyMaterial2; + + // Separate object lists for each re-encryption path to avoid conflicts + private static final List kcGcmObjectsAesToAes = Collections.synchronizedList(new ArrayList<>()); + private static final List kcGcmObjectsAesToRsaCustom = Collections.synchronizedList(new ArrayList<>()); + private static final List kcGcmObjectsAesToRsaDefault = Collections.synchronizedList(new ArrayList<>()); + private static final List kcGcmObjectsRsaToRsa = Collections.synchronizedList(new ArrayList<>()); + private static final List kcGcmObjectsRsaToAesDefault = Collections.synchronizedList(new ArrayList<>()); + + @BeforeAll + static void generateKeys() throws Exception { + KeyGenerator aesKeyGen = KeyGenerator.getInstance("AES"); + aesKeyGen.init(256); + aesKey1 = aesKeyGen.generateKey(); + aesKey2 = aesKeyGen.generateKey(); + + Map aesMatDesc1 = new HashMap<>(); + aesMatDesc1.put("keyId", "aes-key-1"); + aesKeyMaterial1 = KeyMaterial.builder() + .aesKey(ByteBuffer.wrap(aesKey1.getEncoded())) + .materialsDescription(aesMatDesc1) + .build(); + + Map aesMatDesc2 = new HashMap<>(); + aesMatDesc2.put("keyId", "aes-key-2"); + aesKeyMaterial2 = KeyMaterial.builder() + .aesKey(ByteBuffer.wrap(aesKey2.getEncoded())) + .materialsDescription(aesMatDesc2) + .build(); + + KeyPairGenerator rsaKeyGen = KeyPairGenerator.getInstance("RSA"); + rsaKeyGen.initialize(2048); + rsaKeyPair1 = rsaKeyGen.generateKeyPair(); + rsaKeyPair2 = rsaKeyGen.generateKeyPair(); + + Map rsaMatDesc1 = new HashMap<>(); + rsaMatDesc1.put("keyId", "rsa-key-1"); + rsaKeyMaterial1 = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(rsaKeyPair1.getPrivate().getEncoded())) + .materialsDescription(rsaMatDesc1) + .build(); + + Map rsaMatDesc2 = new HashMap<>(); + rsaMatDesc2.put("keyId", "rsa-key-2"); + rsaKeyMaterial2 = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(rsaKeyPair2.getPrivate().getEncoded())) + .materialsDescription(rsaMatDesc2) + .build(); + } + + static List getKcGcmObjectsAesToAes() { return new ArrayList<>(kcGcmObjectsAesToAes); } + static List getKcGcmObjectsAesToRsaCustom() { return new ArrayList<>(kcGcmObjectsAesToRsaCustom); } + static List getKcGcmObjectsAesToRsaDefault() { return new ArrayList<>(kcGcmObjectsAesToRsaDefault); } + static List getKcGcmObjectsRsaToRsa() { return new ArrayList<>(kcGcmObjectsRsaToRsa); } + static List getKcGcmObjectsRsaToAesDefault() { return new ArrayList<>(kcGcmObjectsRsaToAesDefault); } + static KeyMaterial getAesKeyMaterial1() { return aesKeyMaterial1; } + static KeyMaterial getAesKeyMaterial2() { return aesKeyMaterial2; } + static KeyMaterial getRsaKeyMaterial1() { return rsaKeyMaterial1; } + static KeyMaterial getRsaKeyMaterial2() { return rsaKeyMaterial2; } + + public static Stream improvedClientsCanPutRawRSAWithInstructionFile() { + return improvedClientsForTest() + .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + + public static Stream improvedClientsCanPutRawAESWithInstructionFile() { + return improvedClientsForTest() + .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + + @ParameterizedTest(name = "{0}: Encrypt AES objects for AES => AES re-encryption") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$EncryptTests#improvedClientsCanPutRawAESWithInstructionFile") + void encryptAesForAesToAesReencrypt(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(aesKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + "-aes-to-aes-" + language.getLanguageName()), + kcGcmObjectsAesToAes, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Encrypt AES objects for AES => RSA custom suffix re-encryption") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$EncryptTests#improvedClientsCanPutRawAESWithInstructionFile") + void encryptAesForAesToRsaCustomReencrypt(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(aesKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + "-aes-to-rsa-custom-" + language.getLanguageName()), + kcGcmObjectsAesToRsaCustom, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Encrypt AES objects for AES => RSA default suffix re-encryption") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$EncryptTests#improvedClientsCanPutRawAESWithInstructionFile") + void encryptAesForAesToRsaDefaultReencrypt(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(aesKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + "-aes-to-rsa-default-" + language.getLanguageName()), + kcGcmObjectsAesToRsaDefault, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Encrypt RSA objects for RSA => RSA re-encryption") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$EncryptTests#improvedClientsCanPutRawRSAWithInstructionFile") + void encryptRsaForRsaToRsaReencrypt(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(rsaKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + "-rsa-to-rsa-" + language.getLanguageName()), + kcGcmObjectsRsaToRsa, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Encrypt RSA objects for RSA => AES default suffix re-encryption") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$EncryptTests#improvedClientsCanPutRawRSAWithInstructionFile") + void encryptRsaForRsaToAesDefaultReencrypt(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(rsaKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + "-rsa-to-aes-default-" + language.getLanguageName()), + kcGcmObjectsRsaToAesDefault, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @AfterAll + static void signalEncryptionComplete() { + encryptPhaseComplete.countDown(); + } + } + + @Nested + @DisplayName("ReEncryptTests - ReEncrypt") + class ReEncryptTestsNested { + private static List kcGcmObjectsAesToAes, kcGcmObjectsAesToRsaCustom, kcGcmObjectsAesToRsaDefault; + private static List kcGcmObjectsRsaToRsa, kcGcmObjectsRsaToAesDefault; + private static KeyMaterial aesKeyMaterial1, aesKeyMaterial2, rsaKeyMaterial1, rsaKeyMaterial2; + + @BeforeAll + static void setup() throws InterruptedException { + encryptPhaseComplete.await(); + kcGcmObjectsAesToAes = EncryptTests.getKcGcmObjectsAesToAes(); + kcGcmObjectsAesToRsaCustom = EncryptTests.getKcGcmObjectsAesToRsaCustom(); + kcGcmObjectsAesToRsaDefault = EncryptTests.getKcGcmObjectsAesToRsaDefault(); + kcGcmObjectsRsaToRsa = EncryptTests.getKcGcmObjectsRsaToRsa(); + kcGcmObjectsRsaToAesDefault = EncryptTests.getKcGcmObjectsRsaToAesDefault(); + aesKeyMaterial1 = EncryptTests.getAesKeyMaterial1(); + aesKeyMaterial2 = EncryptTests.getAesKeyMaterial2(); + rsaKeyMaterial1 = EncryptTests.getRsaKeyMaterial1(); + rsaKeyMaterial2 = EncryptTests.getRsaKeyMaterial2(); + } + + public static Stream reencryptSupportedClients() { + return improvedClientsForTest() + .filter(target -> RE_ENCRYPT_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + + @ParameterizedTest(name = "{0}: ReEncrypt AES => AES instruction file") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$ReEncryptTestsNested#reencryptSupportedClients") + void reencryptAesToAesInstructionFile(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + + for (int i = 0; i < kcGcmObjectsAesToAes.size(); i++) { + String objectKey = kcGcmObjectsAesToAes.get(i); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(aesKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + ReEncryptOutput response = client.reEncrypt(ReEncryptInput.builder() + .bucket(TestUtils.BUCKET).key(objectKey).clientID(S3ECId) + .newKeyMaterial(aesKeyMaterial2).build()); + + assertNotNull(response); + reEncryptedAesToAes.add(objectKey); + } + } + + @ParameterizedTest(name = "{0}: ReEncrypt RSA => RSA instruction file") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$ReEncryptTestsNested#reencryptSupportedClients") + void reencryptRsaToRsaInstructionFile(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + + for (int i = 0; i < kcGcmObjectsRsaToRsa.size(); i++) { + String objectKey = kcGcmObjectsRsaToRsa.get(i); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(rsaKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + ReEncryptOutput response = client.reEncrypt(ReEncryptInput.builder() + .bucket(TestUtils.BUCKET).key(objectKey).clientID(S3ECId) + .newKeyMaterial(rsaKeyMaterial2).build()); + + assertNotNull(response); + reEncryptedRsaToRsa.add(objectKey); + } + } + + @ParameterizedTest(name = "{0}: ReEncrypt AES => RSA instruction file with custom suffix") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$ReEncryptTestsNested#reencryptSupportedClients") + void reencryptAesToRsaInstructionFile(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + + for (int i = 0; i < kcGcmObjectsAesToRsaCustom.size(); i++) { + String objectKey = kcGcmObjectsAesToRsaCustom.get(i); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(aesKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + ReEncryptOutput response = client.reEncrypt(ReEncryptInput.builder() + .bucket(TestUtils.BUCKET).key(objectKey).clientID(S3ECId) + .newKeyMaterial(rsaKeyMaterial1) + // Java always prepends a `.` + .instructionFileSuffix("instruction-rsa") + .build()); + + assertNotNull(response); + reEncryptedAesToRsa.add(objectKey); + } + } + + @ParameterizedTest(name = "{0}: ReEncrypt RSA => AES instruction file (default suffix)") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$ReEncryptTestsNested#reencryptSupportedClients") + void reencryptRsaToAesDefaultInstructionFile(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + + for (int i = 0; i < kcGcmObjectsRsaToAesDefault.size(); i++) { + String objectKey = kcGcmObjectsRsaToAesDefault.get(i); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(rsaKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + ReEncryptOutput response = client.reEncrypt(ReEncryptInput.builder() + .bucket(TestUtils.BUCKET).key(objectKey).clientID(S3ECId) + .newKeyMaterial(aesKeyMaterial1) + .build()); + + assertNotNull(response); + reEncryptedRsaToAesDefault.add(objectKey); + } + } + + @ParameterizedTest(name = "{0}: ReEncrypt AES => RSA instruction file (default suffix)") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$ReEncryptTestsNested#reencryptSupportedClients") + void reencryptAesToRsaDefaultInstructionFile(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + + for (int i = 0; i < kcGcmObjectsAesToRsaDefault.size(); i++) { + String objectKey = kcGcmObjectsAesToRsaDefault.get(i); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(aesKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + ReEncryptOutput response = client.reEncrypt(ReEncryptInput.builder() + .bucket(TestUtils.BUCKET).key(objectKey).clientID(S3ECId) + .newKeyMaterial(rsaKeyMaterial1) + .build()); + + assertNotNull(response); + reEncryptedAesToRsaDefault.add(objectKey); + } + } + + @AfterAll + static void signalReEncryptionComplete() { + reEncryptPhaseComplete.countDown(); + } + } + + @Nested + @DisplayName("ReEncryptTests - DecryptReEncrypted") + class DecryptReEncryptedTests { + private static KeyMaterial aesKeyMaterial1, aesKeyMaterial2, rsaKeyMaterial1, rsaKeyMaterial2; + + @BeforeAll + static void setup() throws InterruptedException { + reEncryptPhaseComplete.await(); + aesKeyMaterial1 = EncryptTests.getAesKeyMaterial1(); + aesKeyMaterial2 = EncryptTests.getAesKeyMaterial2(); + rsaKeyMaterial1 = EncryptTests.getRsaKeyMaterial1(); + rsaKeyMaterial2 = EncryptTests.getRsaKeyMaterial2(); + } + + public static Stream clientsCanGetRawRSAWithInstructionFile() { + return Stream.concat( + improvedClientsForTest().filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())), + transitionClientsForTest().filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + ); + } + + public static Stream clientsCanGetRawAESWithInstructionFile() { + return Stream.concat( + improvedClientsForTest().filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())), + transitionClientsForTest().filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + ); + } + + public static Stream clientsCanGetRawRSAWithInstructionFileAndCustomSuffix() { + return Stream.concat( + improvedClientsForTest() + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> CUSTOM_INSTRUCTION_SUFFIX_GET_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())), + transitionClientsForTest() + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> CUSTOM_INSTRUCTION_SUFFIX_GET_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + ); + } + + public static Stream clientsCanGetRawAESWithInstructionFileAndCustomSuffix() { + return Stream.concat( + improvedClientsForTest() + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> CUSTOM_INSTRUCTION_SUFFIX_GET_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())), + transitionClientsForTest() + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> CUSTOM_INSTRUCTION_SUFFIX_GET_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + ); + } + + @ParameterizedTest(name = "{0}: Decrypt AES => AES re-encrypted objects") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$DecryptReEncryptedTests#clientsCanGetRawAESWithInstructionFile") + void decryptReencryptedAesToAesObjects(TestUtils.LanguageServerTarget language) { + if (reEncryptedAesToAes.isEmpty()) return; + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder().keyMaterial(aesKeyMaterial2).build()) + .build()); + + // C++ clients require materials description to be passed per-operation + if (language.getLanguageName().startsWith("CPP")) { + TestUtils.DecryptWithMaterialsDescription(client, clientOutput.getClientId(), + reEncryptedAesToAes, aesKeyMaterial2, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } else { + TestUtils.Decrypt(client, clientOutput.getClientId(), reEncryptedAesToAes, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + } + + @ParameterizedTest(name = "{0}: Decrypt RSA => RSA re-encrypted objects") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$DecryptReEncryptedTests#clientsCanGetRawRSAWithInstructionFile") + void decryptReencryptedRsaToRsaObjects(TestUtils.LanguageServerTarget language) { + if (reEncryptedRsaToRsa.isEmpty()) return; + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder().keyMaterial(rsaKeyMaterial2).build()) + .build()); + + // C++ clients require materials description to be passed per-operation + if (language.getLanguageName().startsWith("CPP")) { + TestUtils.DecryptWithMaterialsDescription(client, clientOutput.getClientId(), + reEncryptedRsaToRsa, rsaKeyMaterial2, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } else { + TestUtils.Decrypt(client, clientOutput.getClientId(), reEncryptedRsaToRsa, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + } + + @ParameterizedTest(name = "{0}: Decrypt AES => RSA re-encrypted objects with custom suffix") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$DecryptReEncryptedTests#clientsCanGetRawRSAWithInstructionFileAndCustomSuffix") + void decryptReencryptedAesToRsaObjects(TestUtils.LanguageServerTarget language) { + if (reEncryptedAesToRsa.isEmpty()) return; + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(rsaKeyMaterial1) + .build()) + .build()); + + // C++ clients require materials description to be passed per-operation + if (language.getLanguageName().startsWith("CPP")) { + TestUtils.DecryptWithMaterialsDescription(client, clientOutput.getClientId(), + reEncryptedAesToRsa, rsaKeyMaterial1, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } else { + TestUtils.Decrypt(client, clientOutput.getClientId(), reEncryptedAesToRsa, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + reEncryptedAesToRsa, ".instruction-rsa"); + } + } + + @ParameterizedTest(name = "{0}: Decrypt RSA => AES re-encrypted objects (default suffix)") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$DecryptReEncryptedTests#clientsCanGetRawAESWithInstructionFile") + void decryptReencryptedRsaToAesDefaultObjects(TestUtils.LanguageServerTarget language) { + if (reEncryptedRsaToAesDefault.isEmpty()) return; + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(aesKeyMaterial1) + .build()) + .build()); + + // C++ clients require materials description to be passed per-operation + if (language.getLanguageName().startsWith("CPP")) { + TestUtils.DecryptWithMaterialsDescription(client, clientOutput.getClientId(), + reEncryptedRsaToAesDefault, aesKeyMaterial1, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } else { + TestUtils.Decrypt(client, clientOutput.getClientId(), reEncryptedRsaToAesDefault, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + } + + @ParameterizedTest(name = "{0}: Decrypt AES => RSA re-encrypted objects (default suffix)") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$DecryptReEncryptedTests#clientsCanGetRawRSAWithInstructionFile") + void decryptReencryptedAesToRsaDefaultObjects(TestUtils.LanguageServerTarget language) { + if (reEncryptedAesToRsaDefault.isEmpty()) return; + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(rsaKeyMaterial1) + .build()) + .build()); + + // C++ clients require materials description to be passed per-operation + if (language.getLanguageName().startsWith("CPP")) { + TestUtils.DecryptWithMaterialsDescription(client, clientOutput.getClientId(), + reEncryptedAesToRsaDefault, rsaKeyMaterial1, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } else { + TestUtils.Decrypt(client, clientOutput.getClientId(), reEncryptedAesToRsaDefault, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + } + } +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index 211269d7..e6cfae84 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -6,44 +6,44 @@ package software.amazon.encryption.s3; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import static software.amazon.encryption.s3.TestUtils.*; -import java.net.Socket; -import java.net.URI; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; +import java.security.KeyPair; +import java.security.KeyPairGenerator; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.stream.Stream; +import com.amazonaws.services.s3.AmazonS3EncryptionClientV2; +import com.amazonaws.services.s3.AmazonS3EncryptionV2; +import com.amazonaws.services.s3.model.CryptoConfigurationV2; import com.amazonaws.services.s3.model.KMSEncryptionMaterials; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import software.amazon.smithy.java.aws.client.restjson.RestJsonClientProtocol; -import software.amazon.smithy.java.client.core.ClientConfig; -import software.amazon.smithy.java.client.core.ClientProtocol; -import software.amazon.smithy.java.client.core.endpoint.EndpointResolver; +import org.opentest4j.TestAbortedException; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.encryption.s3.TestUtils.LanguageServerTarget; import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; import software.amazon.encryption.s3.model.CreateClientInput; import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; import software.amazon.encryption.s3.model.GetObjectInput; import software.amazon.encryption.s3.model.GetObjectOutput; +import software.amazon.encryption.s3.model.InstructionFileConfig; import software.amazon.encryption.s3.model.KeyMaterial; import software.amazon.encryption.s3.model.PutObjectInput; import software.amazon.encryption.s3.model.S3ECConfig; -import software.amazon.encryption.s3.model.S3ECTestServerApiService; import software.amazon.encryption.s3.model.S3EncryptionClientError; -import software.amazon.smithy.java.http.api.HttpRequest; -import software.amazon.smithy.java.http.api.HttpResponse; -import com.amazonaws.regions.Region; -import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3Encryption; import com.amazonaws.services.s3.AmazonS3EncryptionClient; import com.amazonaws.services.s3.model.CryptoConfiguration; @@ -53,137 +53,29 @@ import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; public class RoundTripTests { - private static final List serverList; - private static final Map serverMap; - - private static final String KMS_KEY_ARN = System.getenv("TEST_SERVER_KMS_KEY_ARN") != null ? - System.getenv("TEST_SERVER_KMS_KEY_ARN") : "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Test-Server-Github-KMS-Key"; - private static final Region KMS_REGION = Region.getRegion(Regions.fromName("us-west-2")); - private static final String BUCKET = System.getenv("TEST_SERVER_S3_BUCKET") != null ? - System.getenv("TEST_SERVER_S3_BUCKET") : "s3ec-test-server-github-bucket"; - - static { - serverList = new ArrayList<>(2); - serverList.add(new LanguageServerTarget("Java", "8080")); - serverList.add(new LanguageServerTarget("Python", "8081")); - - serverMap = new HashMap<>(2); - serverMap.put("Java", new LanguageServerTarget("Java", "8080")); - serverMap.put("Python", new LanguageServerTarget("Python", "8081")); - } - - static public class LanguageServerTarget { - public String getLanguageName() { - return languageName; - } - - public URI getServerURI() { - return serverURI; - } - - private final String baseURI = "http://localhost"; - private String languageName; - private URI serverURI; - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - LanguageServerTarget that = (LanguageServerTarget) o; - return Objects.equals(languageName, that.languageName) && Objects.equals(serverURI, that.serverURI); - } - - @Override - public int hashCode() { - return Objects.hash(languageName, serverURI); - } - - LanguageServerTarget(String language, String port) { - languageName = language; - serverURI = URI.create(baseURI+ ":" + port); - } - - @Override - public String toString() { - return languageName; - } - } @BeforeAll public static void setup() { - // Wait for servers to start - for (LanguageServerTarget server : serverList) { - if (!serverListening(server.getServerURI())) { - throw new RuntimeException(String.format("Test Server for %s is not running at endpoint: %s", server.getLanguageName(), server.getServerURI())); - } - } - } - - public static boolean serverListening(URI uri) { - try (Socket ignored = new Socket(uri.getHost(), uri.getPort())) { - return true; - } catch (Exception e) { - e.printStackTrace(); - return false; - } - } - - static S3ECTestServerClient testServerClientFor(LanguageServerTarget server) { - S3ECTestServerApiService apiService = S3ECTestServerApiService.instance(); - ClientProtocol rest = new RestJsonClientProtocol(apiService.schema().id()); - return S3ECTestServerClient.builder() - .endpointResolver(EndpointResolver.staticEndpoint(server.serverURI)) - .withConfiguration(ClientConfig.builder() - .service(apiService) - .protocol(rest) - .endpointResolver(EndpointResolver.staticEndpoint(server.serverURI)) - .build()) - .build(); - } - - static Stream clientsForTest() { - return serverList.stream() - .map(LanguageServerTarget::getLanguageName) - .map(Arguments::of); - } - - static Stream crossLanguageClients() { - return serverList.stream() - .flatMap(t1 -> serverList.stream() - .flatMap(t2 -> Stream.of( - Arguments.of(t1, t2) - ))); - } - - /** - * Annoyingly, Smithy doesn't provide an interface for map types - * in HTTP headers, so we have to do the serde ourselves - * Servers need an equivalent utility. - * TODO: Move to a utilities class or something. - */ - private List metadataMapToList(Map md) { - List mdAsList = new ArrayList<>(md.size()); - for (Map.Entry keyValue : md.entrySet()) { - // Using ":" because Smithy will parse "," into a flattened list - mdAsList.add("[" + keyValue.getKey() + "]:[" + keyValue.getValue() + "]"); - } - return mdAsList; + validateServersRunning(); } @ParameterizedTest(name = "{displayName} for Encrypt: {0}, Decrypt: {1}") - @MethodSource("crossLanguageClients") + @MethodSource("software.amazon.encryption.s3.TestUtils#crossLanguageClients") public void crossLanguageTestKms(LanguageServerTarget encLang, LanguageServerTarget decLang) { S3ECTestServerClient encClient = testServerClientFor(encLang); - final String objectKey = "cross-lang-test-key-" + encLang; + final String objectKey = appendTestSuffix("cross-lang-test-key-" + encLang); final String input = "simple-test-input"; KeyMaterial kmsKeyArn = KeyMaterial.builder() .kmsKeyId(KMS_KEY_ARN) .build(); CreateClientOutput encClientOutput = encClient.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn).build()) + .config(S3ECConfig + .builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build() + ) .build()); String encS3ECId = encClientOutput.getClientId(); encClient.putObject(PutObjectInput.builder() @@ -195,7 +87,11 @@ public void crossLanguageTestKms(LanguageServerTarget encLang, LanguageServerTar S3ECTestServerClient decClient = testServerClientFor(decLang); CreateClientOutput decClientOutput = decClient.createClient(CreateClientInput.builder() .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn).build()) + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build() + ) .build()); String decS3ECId = decClientOutput.getClientId(); GetObjectOutput output = decClient.getObject(GetObjectInput.builder() @@ -210,10 +106,13 @@ public void crossLanguageTestKms(LanguageServerTarget encLang, LanguageServerTar } @ParameterizedTest(name = "{displayName} for Encrypt: {0}, Decrypt: {1}") - @MethodSource("crossLanguageClients") + @MethodSource("software.amazon.encryption.s3.TestUtils#crossLanguageClients") public void crossLanguageTestKmsWithEncCtx(LanguageServerTarget encLang, LanguageServerTarget decLang) { + if (ENCRYPTION_CONTEXT_ON_ENCRYPT_UNSUPPORTED.contains(encLang.getLanguageName())) { + return; + } S3ECTestServerClient encClient = testServerClientFor(encLang); - final String objectKey = "cross-lang-test-key-kms-ec-" + encLang; + final String objectKey = appendTestSuffix("cross-lang-test-key-kms-ec-" + encLang); final String input = "simple-test-input"; final Map encCtx = new HashMap<>(); encCtx.put("user-defined-enc-ctx-key", "user-defined-enc-ctx-value"); @@ -224,7 +123,11 @@ public void crossLanguageTestKmsWithEncCtx(LanguageServerTarget encLang, Languag .build(); CreateClientOutput encClientOutput = encClient.createClient(CreateClientInput.builder() .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn).build()) + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build() + ) .build()); String encS3ECId = encClientOutput.getClientId(); @@ -238,7 +141,11 @@ public void crossLanguageTestKmsWithEncCtx(LanguageServerTarget encLang, Languag S3ECTestServerClient decClient = testServerClientFor(decLang); CreateClientOutput decClientOutput = decClient.createClient(CreateClientInput.builder() .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn).build()) + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build() + ) .build()); String decS3ECId = decClientOutput.getClientId(); GetObjectOutput output = decClient.getObject(GetObjectInput.builder() @@ -254,10 +161,16 @@ public void crossLanguageTestKmsWithEncCtx(LanguageServerTarget encLang, Languag } @ParameterizedTest(name = "{displayName} for Encrypt: {0}, Decrypt: {1}") - @MethodSource("crossLanguageClients") - public void crossLanguageTestKmsWithEncCtxMismatchFails(LanguageServerTarget encLang, LanguageServerTarget decLang) { + @MethodSource("software.amazon.encryption.s3.TestUtils#crossLanguageClients") + public void crossLanguageTestKmsWithSubsetEncCtxFails(LanguageServerTarget encLang, LanguageServerTarget decLang) { + if (ENCRYPTION_CONTEXT_ON_DECRYPT_UNSUPPORTED.contains(decLang.getLanguageName())) { + return; + } + if (ENCRYPTION_CONTEXT_ON_ENCRYPT_UNSUPPORTED.contains(encLang.getLanguageName())) { + return; + } S3ECTestServerClient encClient = testServerClientFor(encLang); - final String objectKey = "cross-lang-test-key-kms-ec-mismatch-fails" + encLang; + final String objectKey = appendTestSuffix("cross-lang-test-key-kms-ec-subset-fails" + encLang); final String input = "simple-test-input"; final Map encCtx = new HashMap<>(); encCtx.put("user-defined-enc-ctx-key", "user-defined-enc-ctx-value"); @@ -268,7 +181,10 @@ public void crossLanguageTestKmsWithEncCtxMismatchFails(LanguageServerTarget enc .build(); CreateClientOutput encClientOutput = encClient.createClient(CreateClientInput.builder() .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn).build()) + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) .build()); String encS3ECId = encClientOutput.getClientId(); @@ -282,7 +198,11 @@ public void crossLanguageTestKmsWithEncCtxMismatchFails(LanguageServerTarget enc S3ECTestServerClient decClient = testServerClientFor(decLang); CreateClientOutput decClientOutput = decClient.createClient(CreateClientInput.builder() .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn).build()) + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build() + ) .build()); String decS3ECId = decClientOutput.getClientId(); try { @@ -293,15 +213,83 @@ public void crossLanguageTestKmsWithEncCtxMismatchFails(LanguageServerTarget enc .build()); fail("Expected exception!"); } catch (S3EncryptionClientError e) { - assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3")); + if (decLang.getLanguageName().equals(RUBY_V3) || decLang.getLanguageName().equals(RUBY_V2_TRANSITION)) { + assertTrue(e.getMessage().contains("Value of encryption context from envelope does not match the provided encryption context"), "Actual error: " + e.getMessage()); + } else { + assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3"), "Actual error: " + e.getMessage()); + } + } + } + + @ParameterizedTest(name = "{displayName} for Encrypt: {0}, Decrypt: {1}") + @MethodSource("software.amazon.encryption.s3.TestUtils#crossLanguageClients") + public void crossLanguageTestKmsWithIncorrectEncCtxFails(LanguageServerTarget encLang, LanguageServerTarget decLang) { + if (ENCRYPTION_CONTEXT_ON_DECRYPT_UNSUPPORTED.contains(decLang.getLanguageName())) { + return; + } + S3ECTestServerClient encClient = testServerClientFor(encLang); + final String objectKey = appendTestSuffix("cross-lang-test-key-kms-ec-incorrect-fails" + encLang); + final String input = "simple-test-input"; + final Map encCtx = new HashMap<>(); + encCtx.put("user-defined-enc-ctx-key", "user-defined-enc-ctx-value"); + encCtx.put("user-defined-enc-ctx-key-2", "user-defined-enc-ctx-value-2"); + final List mdAsList = metadataMapToList(encCtx); + KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(KMS_KEY_ARN) + .build(); + CreateClientOutput encClientOutput = encClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build() + ) + .build()); + String encS3ECId = encClientOutput.getClientId(); + + encClient.putObject(PutObjectInput.builder() + .clientID(encS3ECId) + .key(objectKey) + .bucket(BUCKET) + .metadata(mdAsList) + .body(ByteBuffer.wrap(input.getBytes(StandardCharsets.UTF_8))) + .build()); + S3ECTestServerClient decClient = testServerClientFor(decLang); + CreateClientOutput decClientOutput = decClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build() + ) + .build()); + String decS3ECId = decClientOutput.getClientId(); + + final Map incorrectEncCtx = new HashMap<>(); + incorrectEncCtx.put("this-is-wrong-ec-key", "bad-value"); + var incorrectMdAsList = metadataMapToList(incorrectEncCtx); + try { + decClient.getObject(GetObjectInput.builder() + .clientID(decS3ECId) + .bucket(BUCKET) + .key(objectKey) + .metadata(incorrectMdAsList) + .build()); + fail("Expected exception!"); + } catch (S3EncryptionClientError e) { + if (decLang.getLanguageName().equals(RUBY_V3) || decLang.getLanguageName().equals(RUBY_V2_TRANSITION)) { + assertTrue(e.getMessage().contains("Value of encryption context from envelope does not match the provided encryption context"), "Actual error: " + e.getMessage()); + } else { + assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3"), "Actual error: " + e.getMessage()); + } } } @ParameterizedTest(name = "{displayName} for Encrypt: Java, Decrypt: {0}") - @MethodSource("clientsForTest") - public void kmsV1Legacy(String language) { - S3ECTestServerClient client = testServerClientFor(serverMap.get(language)); - final String objectKey = "test-key-kms-v1-" + language; + @MethodSource("software.amazon.encryption.s3.TestUtils#clientsForTest") + public void kmsV1Legacy(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = testServerClientFor(language); + final String objectKey = appendTestSuffix("test-key-kms-v1-" + language); final String input = "simple-test-input"; KeyMaterial kmsKeyArn = KeyMaterial.builder() .kmsKeyId(KMS_KEY_ARN) @@ -310,6 +298,8 @@ public void kmsV1Legacy(String language) { .config(S3ECConfig.builder() .enableLegacyWrappingAlgorithms(true) .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) .build()) .build()); String s3ECId = output1.getClientId(); @@ -340,10 +330,10 @@ public void kmsV1Legacy(String language) { } @ParameterizedTest(name = "{displayName} for Encrypt: Java, Decrypt: {0}") - @MethodSource("clientsForTest") - public void kmsV1LegacyWithEncCtx(String language) { - S3ECTestServerClient client = testServerClientFor(serverMap.get(language)); - final String objectKey = "test-key-kms-v1-with-enc-ctx-" + language; + @MethodSource("software.amazon.encryption.s3.TestUtils#clientsForTest") + public void kmsV1LegacyWithEncCtx(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = testServerClientFor(language); + final String objectKey = appendTestSuffix("test-key-kms-v1-with-enc-ctx-" + language); final String input = "simple-test-input"; KeyMaterial kmsKeyArn = KeyMaterial.builder() .kmsKeyId(KMS_KEY_ARN) @@ -352,6 +342,8 @@ public void kmsV1LegacyWithEncCtx(String language) { .config(S3ECConfig.builder() .enableLegacyWrappingAlgorithms(true) .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) .build()) .build()); String s3ECId = output1.getClientId(); @@ -389,10 +381,10 @@ public void kmsV1LegacyWithEncCtx(String language) { } @ParameterizedTest(name = "{displayName} for Encrypt: Java, Decrypt: {0}") - @MethodSource("clientsForTest") - public void kmsV1LegacyFailsWhenLegacyDisabled(String language) { - S3ECTestServerClient client = testServerClientFor(serverMap.get(language)); - final String objectKey = "test-key-kms-v1-fails-disabled" + language; + @MethodSource("software.amazon.encryption.s3.TestUtils#clientsForTest") + public void kmsV1LegacyFailsWhenLegacyDisabled(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = testServerClientFor(language); + final String objectKey = appendTestSuffix("test-key-kms-v1-fails-disabled" + language); final String input = "simple-test-input"; KeyMaterial kmsKeyArn = KeyMaterial.builder() .kmsKeyId(KMS_KEY_ARN) @@ -401,6 +393,8 @@ public void kmsV1LegacyFailsWhenLegacyDisabled(String language) { .config(S3ECConfig.builder() .enableLegacyWrappingAlgorithms(false) .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) .build()) .build()); String s3ECId = output1.getClientId(); @@ -429,8 +423,261 @@ public void kmsV1LegacyFailsWhenLegacyDisabled(String language) { .build()); fail("Expected Exception"); } catch (S3EncryptionClientError e) { - assertTrue(e.getMessage().contains("Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms")); + if (language.getLanguageName().equals(NET_V3_TRANSITION) || language.getLanguageName().equals(NET_V4) + || language.getLanguageName().equals(CPP_V2_TRANSITION) || language.getLanguageName().equals(CPP_V3)) { + assertTrue(e.getMessage().contains( + "The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration" + ), "Actual error:" + e.getMessage()); + } else if (language.getLanguageName().equals(RUBY_V3) || language.getLanguageName().equals(RUBY_V2_TRANSITION)) { + assertTrue(e.getMessage().contains( + "The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration security_profile = :v2. Retry with :v2_and_legacy or re-encrypt the object." + ), "Actual error:" + e.getMessage()); + } else if (language.getLanguageName().equals(PHP_V3)) { + assertTrue(e.getMessage().contains("The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration @SecurityProfile=V3. Retry with V3_AND_LEGACY enabled or reencrypt the object."), "Actual error: " + e.getMessage()); + } else { + assertTrue(e.getMessage().contains("Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms"), "Actual error: " + e.getMessage()); + } + } + } + + @ParameterizedTest(name = "{displayName} for Encrypt: {0}, Decrypt: {1}") + @MethodSource("software.amazon.encryption.s3.TestUtils#crossLanguageClients") + public void rsaRoundTrip(LanguageServerTarget encLang, LanguageServerTarget decLang) throws Exception { + if (!RAW_SUPPORTED.contains(encLang.getLanguageName())) { + throw new TestAbortedException("not encrypting raw keyrings with: " + encLang.getLanguageName()); + } + if (!RAW_SUPPORTED.contains(decLang.getLanguageName())) { + throw new TestAbortedException("not decrypting raw keyrings with: " + decLang.getLanguageName()); + } + S3ECTestServerClient encClient = testServerClientFor(encLang); + S3ECTestServerClient decClient = testServerClientFor(decLang); + final String objectKey = appendTestSuffix(String.format("rsa-write-%s-read-%s", encLang.getLanguageName(), decLang.getLanguageName())); + final String input = "simple-test-input-rsa"; + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(2048); + KeyPair RSA_KEY_PAIR_1 = keyPairGen.generateKeyPair(); + + KeyMaterial rsaKeyOne = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) + .build(); + CreateClientOutput encClientOutput = encClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + // TODO: use this for now to satisfy current. think about long term soln for this + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .keyMaterial(rsaKeyOne).build()) + .build()); + String encS3ECId = encClientOutput.getClientId(); + CreateClientOutput decClientOutput = decClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .keyMaterial(rsaKeyOne).build()) + .build()); + String decS3ECId = decClientOutput.getClientId(); + encClient.putObject(PutObjectInput.builder() + .clientID(encS3ECId) + .key(objectKey) + .bucket(BUCKET) + .body(ByteBuffer.wrap(input.getBytes(StandardCharsets.UTF_8))) + .build()); + GetObjectOutput output = decClient.getObject(GetObjectInput.builder() + .clientID(decS3ECId) + .bucket(BUCKET) + .key(objectKey) + .build()); + assertEquals(input, new String(output.getBody().array())); + } + + @ParameterizedTest(name = "{displayName} for Encrypt: Java, Decrypt: {0}") + @MethodSource("software.amazon.encryption.s3.TestUtils#clientsForTest") + public void instructionFileReadV2Format(TestUtils.LanguageServerTarget language) { + if (KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(language.getLanguageName())) { + throw new TestAbortedException(String.format("%s does not support KMS instruction files", language.getLanguageName())); } + if (INSTRUCTION_FILE_GET_UNSUPPORTED.contains(language.getLanguageName())) { + throw new TestAbortedException(String.format("%s does not support instruction file Gets", language.getLanguageName())); + } + S3ECTestServerClient client = testServerClientFor(language); + final String objectKey = appendTestSuffix("read-instruction-file-v2-" + language); + final String input = "simple-test-input"; + KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(KMS_KEY_ARN) + .build(); + CreateClientOutput output1 = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .enableLegacyWrappingAlgorithms(true) + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String s3ECId = output1.getClientId(); + + // Write with instruction file using V2 client + EncryptionMaterialsProvider materialsProvider = new KMSEncryptionMaterialsProvider(KMS_KEY_ARN); + CryptoConfigurationV2 cryptoConfigurationV2 = new CryptoConfigurationV2(); + cryptoConfigurationV2.setStorageMode(CryptoStorageMode.InstructionFile); + AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() + .withEncryptionMaterialsProvider(materialsProvider) + .withCryptoConfiguration(cryptoConfigurationV2) + .build(); + v2Client.putObject(BUCKET, objectKey, input); + + // Read should be enabled by default + GetObjectOutput output = client.getObject(GetObjectInput.builder() + .clientID(s3ECId) + .bucket(BUCKET) + .key(objectKey) + .build()); + + assertEquals(input, new String(output.getBody().array())); } + @ParameterizedTest(name = "{displayName} for Encrypt: {0}, Decrypt: {1}") + @MethodSource("software.amazon.encryption.s3.TestUtils#crossLanguageClients") + public void instructionFileWriteAndRead(LanguageServerTarget encLang, LanguageServerTarget decLang) throws Exception { + if (INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(encLang.getLanguageName())) { + throw new TestAbortedException("not testing " + encLang.getLanguageName()); + } + if (INSTRUCTION_FILE_GET_UNSUPPORTED.contains(decLang.getLanguageName())) { + throw new TestAbortedException("not testing " + encLang.getLanguageName()); + } + if (KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(encLang.getLanguageName())) { + throw new TestAbortedException("not testing " + encLang.getLanguageName()); + } + if (KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(decLang.getLanguageName())) { + throw new TestAbortedException("not testing " + encLang.getLanguageName()); + } + S3ECTestServerClient encClient = testServerClientFor(encLang); + S3ECTestServerClient decClient = testServerClientFor(decLang); + final String objectKey = appendTestSuffix(String.format("write-%s-read-%s-instruction-file", encLang.getLanguageName(), decLang.getLanguageName())); + final String input = "simple-test-input"; + KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(KMS_KEY_ARN) + .build(); + CreateClientOutput encOutput = encClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String encS3ECId = encOutput.getClientId(); + CreateClientOutput decOutput = decClient.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String decS3ECId = decOutput.getClientId(); + + // Write with instruction file + encClient.putObject(PutObjectInput.builder() + .clientID(encS3ECId) + .bucket(BUCKET) + .key(objectKey) + .body(ByteBuffer.wrap(input.getBytes(StandardCharsets.UTF_8))) + .build()); + + // Assert using Java plaintext client that an instruction file exists + ResponseBytes ptInstFile; + try (S3Client ptS3Client = S3Client.create()) { + ptInstFile = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); + } + // Check for inst file key + if (!encLang.getLanguageName().startsWith("Ruby") && !encLang.getLanguageName().startsWith("PHP")) { + // Ruby and PHP do not include it :( + assertTrue(ptInstFile.response().metadata().containsKey("x-amz-crypto-instr-file")); + } + + // At high concurrency, this test tends to get: + // BadDigest Message: The CRC64NVME you specified did not match the calculated checksum. + // I think this is a read after write issue. + // A better fix, would be to break this tests suite up into encrypt/decrypt + // rather than having a test for many pairs and doing encrypt/decrypt on each pair + Thread.sleep(100); + + assertFalse(ptInstFile.asUtf8String().isEmpty()); + // Read should be enabled by default + GetObjectOutput output = decClient.getObject(GetObjectInput.builder() + .clientID(decS3ECId) + .bucket(BUCKET) + .key(objectKey) + .build()); + + assertEquals(input, new String(output.getBody().array())); + } + + @ParameterizedTest(name = "{displayName} for Encrypt: {0}, Decrypt: {1}") + @MethodSource("software.amazon.encryption.s3.TestUtils#crossLanguageClients") + public void instructionFileWriteAndReadWithRSA(LanguageServerTarget encLang, LanguageServerTarget decLang) throws Exception { + // Early validation + if (!RAW_SUPPORTED.contains(encLang.getLanguageName())) { + throw new TestAbortedException("not encrypting raw keyring with: " + encLang.getLanguageName()); + } + if (!RAW_SUPPORTED.contains(decLang.getLanguageName())) { + throw new TestAbortedException("not decrypting raw keyring with: " + decLang.getLanguageName()); + } + + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(2048); + KeyMaterial rsaKeyMaterial = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(keyPairGen.generateKeyPair().getPrivate().getEncoded())) + .build(); + + S3ECConfig config = S3ECConfig.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .keyMaterial(rsaKeyMaterial) + .build(); + + // Create clients + S3ECTestServerClient encClient = testServerClientFor(encLang); + S3ECTestServerClient decClient = testServerClientFor(decLang); + + String encS3ECId = encClient.createClient(CreateClientInput.builder().config(config).build()).getClientId(); + String decS3ECId = decClient.createClient(CreateClientInput.builder().config(config).build()).getClientId(); + + final String objectKey = appendTestSuffix(String.format("rsa-insfile-write-%s-read-%s", + encLang.getLanguageName(), decLang.getLanguageName())); + final String input = "simple-test-input-rsa"; + + // Encrypt + encClient.putObject(PutObjectInput.builder() + .clientID(encS3ECId) + .bucket(BUCKET) + .key(objectKey) + .body(ByteBuffer.wrap(input.getBytes(StandardCharsets.UTF_8))) + .build()); + + // Assert using Java plaintext client that an instruction file exists + ResponseBytes ptInstFile; + try (S3Client ptS3Client = S3Client.create()) { + ptInstFile = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); + } + // assertTrue(ptInstFile.response().metadata().containsKey("x-amz-crypto-instr-file")); + assertFalse(ptInstFile.asUtf8String().isEmpty()); + // Read should be enabled by default + GetObjectOutput output = decClient.getObject(GetObjectInput.builder() + .clientID(decS3ECId) + .bucket(BUCKET) + .key(objectKey) + .build()); + + assertEquals(input, new String(output.getBody().array())); + } } diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java new file mode 100644 index 00000000..2b9cd062 --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -0,0 +1,812 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.encryption.s3; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +import java.net.Socket; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.amazonaws.services.s3.model.S3Object; +import com.fasterxml.jackson.databind.ObjectMapper; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.ObjectMetadata; +import org.joda.time.DateTime; +import org.joda.time.format.DateTimeFormat; +import org.junit.jupiter.params.provider.Arguments; +import com.amazonaws.regions.Region; +import com.amazonaws.regions.Regions; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.GetObjectInput; +import software.amazon.encryption.s3.model.GetObjectOutput; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.PutObjectInput; +import software.amazon.encryption.s3.model.PutObjectOutput; +import software.amazon.encryption.s3.model.S3EncryptionClientError; +import software.amazon.smithy.java.aws.client.restjson.RestJsonClientProtocol; +import software.amazon.smithy.java.client.core.ClientConfig; +import software.amazon.smithy.java.client.core.ClientProtocol; +import software.amazon.smithy.java.client.core.endpoint.EndpointResolver; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.S3ECTestServerApiService; +import software.amazon.smithy.java.http.api.HttpRequest; +import software.amazon.smithy.java.http.api.HttpResponse; + +public class TestUtils { + + // Version name constants + // Each language can have up to 3 versions: + // vN-Current: Currently released version. Does not support setting commitment policy. + // vN-Transition: Proposed feature release version. Supports reading messages encrypted with key commitment. + // vN+1: Proposed breaking release version. Supports reading/writing messages encrypted with key commitment. + + public static final String JAVA_V3_TRANSITION = "Java-V3-Transition"; + public static final String JAVA_V4 = "Java-V4"; + + // No Python S3EC versions are released. Only test V3 as the "vN+1" version. + public static final String PYTHON_V3 = "Python-V3"; + + public static final String GO_V3_TRANSITION = "Go-V3-Transition"; + public static final String GO_V4 = "Go-V4"; + + public static final String NET_V2_TRANSITION = "NET-V2-Transition"; + public static final String NET_V3_TRANSITION = "NET-V3-Transition"; + public static final String NET_V4 = "NET-V4"; + + public static final String CPP_V2_TRANSITION = "CPP-V2-Transition"; + public static final String CPP_V3 = "CPP-V3"; + + public static final String RUBY_V2_TRANSITION = "Ruby-V2-Transition"; + public static final String RUBY_V3 = "Ruby-V3"; + + public static final String PHP_V2_TRANSITION = "PHP-V2-Transition"; + public static final String PHP_V3 = "PHP-V3"; + + // Test configuration constants + public static final String KMS_KEY_ARN = System.getenv("TEST_SERVER_KMS_KEY_ARN") != null ? + System.getenv("TEST_SERVER_KMS_KEY_ARN") : "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Test-Server-Github-KMS-Key"; + public static final Region KMS_REGION = Region.getRegion(Regions.fromName("us-west-2")); + public static final String BUCKET = System.getenv("TEST_SERVER_S3_BUCKET") != null ? + System.getenv("TEST_SERVER_S3_BUCKET") : "s3ec-test-server-github-bucket"; + + // Sets of unsupported features by language + public static final Set ENCRYPTION_CONTEXT_ON_DECRYPT_UNSUPPORTED = + Set.of(PHP_V2_TRANSITION, PHP_V3, NET_V3_TRANSITION, NET_V4); + + public static final Set ENCRYPTION_CONTEXT_ON_ENCRYPT_UNSUPPORTED = + Set.of(NET_V3_TRANSITION, NET_V4); + + public static final Set RE_ENCRYPT_SUPPORTED = + Set.of(JAVA_V3_TRANSITION, JAVA_V4); + + public static final Set RANGED_GETS_SUPPORTED = + Set.of( + JAVA_V3_TRANSITION, JAVA_V4, CPP_V2_TRANSITION, CPP_V3 + ); + + // Cpp only supports Raw AES + public static final Set RAW_AES_SUPPORTED = + Set.of(JAVA_V3_TRANSITION, JAVA_V4, NET_V3_TRANSITION, NET_V4, RUBY_V2_TRANSITION, RUBY_V3, CPP_V2_TRANSITION, CPP_V3); + + public static final Set RAW_RSA_SUPPORTED = + Set.of(JAVA_V3_TRANSITION, JAVA_V4, NET_V3_TRANSITION, NET_V4, RUBY_V2_TRANSITION, RUBY_V3); + + // Intersection of RAW_AES_SUPPORTED and RAW_RSA_SUPPORTED + public static final Set RAW_SUPPORTED = + RAW_AES_SUPPORTED.stream() + .filter(RAW_RSA_SUPPORTED::contains) + .collect(Collectors.toSet()); + + // .NET only supports decrypting instruction files using AES and RSA. + // Python MUST support decrypting KMS instruction files, but does not yet. + public static final Set KMS_INSTRUCTION_FILE_UNSUPPORTED = + Set.of(NET_V2_TRANSITION, NET_V3_TRANSITION, NET_V4); + + // Go does not write with instruction files + public static final Set INSTRUCTION_FILE_PUT_UNSUPPORTED = + Set.of(GO_V3_TRANSITION, GO_V4, PYTHON_V3); + + // Not implemented yet in Python. + public static final Set INSTRUCTION_FILE_GET_UNSUPPORTED = + Set.of(PYTHON_V3); + + // Languages that support custom instruction file suffix on GetObject + // Only Java, Ruby, and PHP servers have been updated with this feature + // This is a current gap. + public static final Set CUSTOM_INSTRUCTION_SUFFIX_GET_SUPPORTED = + Set.of( + JAVA_V3_TRANSITION, + JAVA_V4, + RUBY_V2_TRANSITION, + RUBY_V3, + PHP_V2_TRANSITION, + PHP_V3 + ); + + public static final Set TRANSITION_VERSIONS = + Set.of( + JAVA_V3_TRANSITION, + GO_V3_TRANSITION, + NET_V3_TRANSITION, + CPP_V2_TRANSITION, + PHP_V2_TRANSITION, + RUBY_V2_TRANSITION + ); + + public static final Set IMPROVED_VERSIONS = + Set.of( + JAVA_V4, + // PYTHON_V3, + GO_V4, + NET_V4, + CPP_V3, + PHP_V3, + RUBY_V3 + ); + + private static final Map serverMap; + + static { + final Map servers = new LinkedHashMap<>(); + servers.put(PYTHON_V3, new LanguageServerTarget(PYTHON_V3, "8081")); + servers.put(CPP_V2_TRANSITION, new LanguageServerTarget(CPP_V2_TRANSITION, "8097")); + servers.put(CPP_V3, new LanguageServerTarget(CPP_V3, "8091")); + servers.put(GO_V4, new LanguageServerTarget(GO_V4, "8089")); + servers.put(NET_V4, new LanguageServerTarget(NET_V4, "8090")); + servers.put(RUBY_V3, new LanguageServerTarget(RUBY_V3, "8092")); + servers.put(PHP_V3, new LanguageServerTarget(PHP_V3, "8093")); + servers.put(JAVA_V3_TRANSITION, new LanguageServerTarget(JAVA_V3_TRANSITION, "8094")); + servers.put(GO_V3_TRANSITION, new LanguageServerTarget(GO_V3_TRANSITION, "8095")); + servers.put(RUBY_V2_TRANSITION, new LanguageServerTarget(RUBY_V2_TRANSITION, "8098")); + servers.put(PHP_V2_TRANSITION, new LanguageServerTarget(PHP_V2_TRANSITION, "8099")); + servers.put(JAVA_V4, new LanguageServerTarget(JAVA_V4, "8088")); + servers.put(NET_V3_TRANSITION, new LanguageServerTarget(NET_V3_TRANSITION, "8100")); + serverMap = filterServers(servers); + + System.out.println("=== Configured Test Servers ==="); + System.out.println("\nServers:"); + serverMap.forEach((name, target) -> { + System.out.println(" " + name + " -> " + target.getServerURI()); + }); + System.out.println("\nTotal servers configured: " + serverMap.size()); + System.out.println("================================"); + } + + public static class LanguageServerTarget { + private final String baseURI = "http://localhost"; + private String languageName; + private URI serverURI; + + public LanguageServerTarget(String language, String port) { + languageName = language; + serverURI = URI.create(baseURI + ":" + port); + } + + public String getLanguageName() { + return languageName; + } + + public URI getServerURI() { + return serverURI; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + LanguageServerTarget that = (LanguageServerTarget) o; + return Objects.equals(languageName, that.languageName) && Objects.equals(serverURI, that.serverURI); + } + + @Override + public int hashCode() { + return Objects.hash(languageName, serverURI); + } + + @Override + public String toString() { + return languageName; + } + } + + /** + * Filters the available servers based on system property test.filter.servers + * @param allServers Map of all available servers + * @return Filtered map of servers to use for testing + */ + private static Map filterServers(Map allServers) { + final String maybeFilter = System.getProperty("test.filter.servers"); + if (maybeFilter == null || maybeFilter.trim().isEmpty()) { + return allServers; // No filtering - use all servers + } + + System.out.println("Filtering with: " + maybeFilter); + + final String[] filters = Arrays.stream(maybeFilter.split(",")) + .map(String::trim) + .map(String::toLowerCase) + .toArray(String[]::new); + + return allServers.entrySet().stream() + .filter(entry -> { + String key = entry.getKey().toLowerCase(); + System.out.println("Checking server name:" + key); + return Arrays.stream(filters).anyMatch(key::contains); + }) + .collect(Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue, + (e1, e2) -> e1, // merge function (not really needed) + LinkedHashMap::new // preserve order + )); + } + + /** + * Gets the map of available server targets for testing + * @return Map of language names to server targets + */ + public static Map getServerMap() { + return serverMap; + } + + /** + * Checks if a server is listening on the specified URI + * @param uri The URI to check + * @return true if server is listening, false otherwise + */ + public static boolean serverListening(URI uri) { + try (Socket ignored = new Socket(uri.getHost(), uri.getPort())) { + return true; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + + /** + * Creates a test server client for the specified language server target + * @param server The language server target + * @return Configured S3ECTestServerClient + */ + public static S3ECTestServerClient testServerClientFor(LanguageServerTarget server) { + S3ECTestServerApiService apiService = S3ECTestServerApiService.instance(); + ClientProtocol rest = new RestJsonClientProtocol(apiService.schema().id()); + return S3ECTestServerClient.builder() + .endpointResolver(EndpointResolver.staticEndpoint(server.serverURI)) + .withConfiguration(ClientConfig.builder() + .service(apiService) + .protocol(rest) + .endpointResolver(EndpointResolver.staticEndpoint(server.serverURI)) + .build()) + .build(); + } + + /** + * Converts a metadata map to a list format for Smithy serialization + * Annoyingly, Smithy doesn't provide an interface for map types + * in HTTP headers, so we have to do the serde ourselves + * @param md The metadata map + * @return List representation of the metadata + */ + public static List metadataMapToList(Map md) { + List mdAsList = new ArrayList<>(md.size()); + for (Map.Entry keyValue : md.entrySet()) { + // Using ":" because Smithy will parse "," into a flattened list + mdAsList.add("[" + keyValue.getKey() + "]:[" + keyValue.getValue() + "]"); + } + return mdAsList; + } + + /** + * Validates that all servers in the server map are running + * @throws RuntimeException if any server is not running + */ + public static void validateServersRunning() { + for (LanguageServerTarget server : serverMap.values()) { + if (!serverListening(server.getServerURI())) { + throw new RuntimeException(String.format("Test Server for %s is not running at endpoint: %s", + server.getLanguageName(), server.getServerURI())); + } + } + } + + /** + * Provides a stream of arguments for parameterized tests that test individual clients + * @return Stream of Arguments containing language names for testing + */ + public static Stream clientsForTest() { + return serverMap.values().stream() + .map(Arguments::of); + } + + /** + * Get stream of arguments for transition version clients for testing. + */ + public static Stream transitionClientsForTest() { + return serverMap.values().stream() + .filter(target -> TRANSITION_VERSIONS.contains(target.getLanguageName())) + .map(Arguments::of); + } + + /** + * Get stream of arguments for improved version clients for testing. + */ + public static Stream improvedClientsForTest() { + return serverMap.values().stream() + .filter(target -> IMPROVED_VERSIONS.contains(target.getLanguageName())) + .map(Arguments::of); + } + + /** + * Get stream of arguments for clients that support RAW AES (includes CPP). + */ + public static Stream clientsRawAesForTest() { + Stream improved = improvedClientsForTest() + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + Stream transition = transitionClientsForTest() + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + return Stream.concat(improved, transition); + } + + /** + * Get stream of arguments for clients that support RAW RSA (excludes CPP). + */ + public static Stream clientsRawRsaForTest() { + Stream improved = improvedClientsForTest() + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + Stream transition = transitionClientsForTest() + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + return Stream.concat(improved, transition); + } + + /** + * These functions provide a stream of arguments for parameterized tests. + * @return Stream of Arguments containing pairs of LanguageServerTarget for encryption and decryption + */ + public static Stream encryptImprovedDecryptImproved() { + return improvedClientsForTest() + .flatMap(encrypt -> improvedClientsForTest() + .flatMap(decrypt -> Stream.of( + Arguments.of(encrypt.get()[0], decrypt.get()[0]) + ))); + } + + public static Stream encryptImprovedDecryptTransition() { + return improvedClientsForTest() + .flatMap(encrypt -> transitionClientsForTest() + .flatMap(decrypt -> Stream.of( + Arguments.of(encrypt.get()[0], decrypt.get()[0]) + ))); + } + + public static Stream encryptTransitionDecryptImproved() { + return transitionClientsForTest() + .flatMap(encrypt -> improvedClientsForTest() + .flatMap(decrypt -> Stream.of( + Arguments.of(encrypt.get()[0], decrypt.get()[0]) + ))); + } + + /** + * Provides a stream of arguments for parameterized tests that test cross-language compatibility + * @return Stream of Arguments containing pairs of LanguageServerTarget for encryption and decryption + */ + public static Stream crossLanguageClients() { + return serverMap.values().stream() + .flatMap(t1 -> serverMap.values().stream() + .flatMap(t2 -> Stream.of( + Arguments.of(t1, t2) + ))); + } + + /** + * For a given string, append a suffix to distinguish it from + * simultaneous test runs. + * @param s The string to append the suffix to + * @return The string with the suffix appended + */ + public static String appendTestSuffix(final String s) { + StringBuilder stringBuilder = new StringBuilder(s); + stringBuilder.append(DateTimeFormat.forPattern("-yyMMdd-hhmmss-").print(new DateTime())); + stringBuilder.append((int) (Math.random() * 100000)); + return stringBuilder.toString(); + } + + private static AmazonS3 s3Client = AmazonS3ClientBuilder.defaultClient(); + public static EncryptionAlgorithm GetEncryptionAlgorithm(String objectKey) + { + // Lambda to determine encryption algorithm from a metadata map + java.util.function.Function, Optional> getAlgorithmFromMap = (map) -> { + if (map.containsKey("x-amz-c")) { + return Optional.of(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } else if (map.containsKey("x-amz-cek-alg")) { + String cek = (String) map.get("x-amz-cek-alg"); + if (cek.contains("CBC")) { + return Optional.of(EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF); + } else if (cek.contains("GCM")) { + return Optional.of(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + } + return Optional.empty(); + }; + + ObjectMetadata metadata = s3Client.getObjectMetadata(TestUtils.BUCKET, objectKey); + Map userMetadata = metadata.getUserMetadata(); + + // Try to get algorithm from object metadata + Optional algorithm = getAlgorithmFromMap.apply(userMetadata); + if (algorithm.isPresent()) { + return algorithm.get(); + } + + // Check instruction file + try { + String instructionFileKey = objectKey + ".instruction"; + com.amazonaws.services.s3.model.S3Object instructionFileObject = + s3Client.getObject(TestUtils.BUCKET, instructionFileKey); + + // Read instruction file content + java.io.InputStream inputStream = instructionFileObject.getObjectContent(); + String instructionFileJson = new String( + inputStream.readAllBytes(), + java.nio.charset.StandardCharsets.UTF_8 + ); + inputStream.close(); + + // Parse JSON to get metadata + com.fasterxml.jackson.databind.ObjectMapper mapper = new com.fasterxml.jackson.databind.ObjectMapper(); + Map instructionFileMap = mapper.readValue(instructionFileJson, Map.class); + + // Try to get algorithm from instruction file + algorithm = getAlgorithmFromMap.apply(instructionFileMap); + if (algorithm.isPresent()) { + return algorithm.get(); + } + } catch (Exception e) { + // Instruction file doesn't exist or couldn't be read + } + + throw new RuntimeException("Could not determine encryption algorithm from object metadata or instruction file!"); + } + + public static void Encrypt( + S3ECTestServerClient client, + String S3ECId, + String objectKey, + List crossLanguageObjects, + EncryptionAlgorithm expectedEncryptionAlgorithm + ) { + PutObjectOutput foo = client.putObject(PutObjectInput.builder() + .clientID(S3ECId) + .key(objectKey) + .bucket(TestUtils.BUCKET) + .body(ByteBuffer.wrap(objectKey.getBytes(StandardCharsets.UTF_8))) + .build()); + + assertEquals( + expectedEncryptionAlgorithm, + GetEncryptionAlgorithm(objectKey), + "When encrypting the EncryptionAlgorithm does not match the expected value: " + expectedEncryptionAlgorithm + ); + + crossLanguageObjects.add(objectKey); + } + + public static void Decrypt( + S3ECTestServerClient client, + String S3ECId, + List crossLanguageObjects, + EncryptionAlgorithm expectedEncryptionAlgorithm + ) { + // Call 5-parameter version with crossLanguageObjects as expectedPlaintexts + Decrypt(client, S3ECId, crossLanguageObjects, expectedEncryptionAlgorithm, crossLanguageObjects); + } + + public static void Decrypt( + S3ECTestServerClient client, + String S3ECId, + List crossLanguageObjects, + EncryptionAlgorithm expectedEncryptionAlgorithm, + List expectedPlaintexts + ) { + Decrypt(client, S3ECId, crossLanguageObjects, expectedEncryptionAlgorithm, expectedPlaintexts, null); + } + + public static void Decrypt( + S3ECTestServerClient client, + String S3ECId, + List crossLanguageObjects, + EncryptionAlgorithm expectedEncryptionAlgorithm, + List expectedPlaintexts, + String instructionFileSuffix + ) { + if (crossLanguageObjects.isEmpty()) { + fail("There is nothing to decrypt"); + } + + List failures = new ArrayList<>(); + for (int i = 0; i < crossLanguageObjects.size(); i++) { + try { + String objectKey = crossLanguageObjects.get(i); + String expectedPlaintext = expectedPlaintexts.get(i); + + GetObjectInput.Builder builder = GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey); + + // Add custom instruction file suffix if provided + if (instructionFileSuffix != null && !instructionFileSuffix.isEmpty()) { + builder.instructionFileSuffix(instructionFileSuffix); + } + + GetObjectOutput output = client.getObject(builder.build()); + + // Then: Pass + assertEquals(expectedPlaintext, new String(output.getBody().array())); + assertEquals( + expectedEncryptionAlgorithm, + GetEncryptionAlgorithm(objectKey), + "When decrypting the EncryptionAlgorithm does not match the expected value: " + expectedEncryptionAlgorithm + ); + } catch (Exception e) { + failures.add(String.format( + "Failed to decrypt object '%s' (index %d): %s - %s", + crossLanguageObjects.get(i), i, e.getClass().getSimpleName(), e.getMessage() + )); + } + } + + if (!failures.isEmpty()) { + throw new AssertionError(String.format( + "Decryption failed for %d out of %d objects:\n%s", + failures.size(), crossLanguageObjects.size(), + String.join("\n", failures) + )); + } + } + + /** + * Decrypt helper for C++ clients that require materials description per-operation. + * + * C++ SDK Design: Unlike Java/. NET/etc where materials description is embedded in the + * keyring during client creation, the C++ SDK requires passing materials description + * as a contextMap parameter to each GetObject/PutObject operation. + * + * This helper extracts materials description from KeyMaterial and passes it via the + * Content-Metadata header on each GetObject call, which the C++ server converts to + * the contextMap parameter required by the C++ SDK. + */ + public static void DecryptWithMaterialsDescription( + S3ECTestServerClient client, + String S3ECId, + List crossLanguageObjects, + KeyMaterial keyMaterial, + EncryptionAlgorithm expectedEncryptionAlgorithm + ) { + DecryptWithMaterialsDescription(client, S3ECId, crossLanguageObjects, keyMaterial, + expectedEncryptionAlgorithm, crossLanguageObjects); + } + + /** + * Decrypt helper for C++ clients with custom expected plaintexts. + */ + public static void DecryptWithMaterialsDescription( + S3ECTestServerClient client, + String S3ECId, + List crossLanguageObjects, + KeyMaterial keyMaterial, + EncryptionAlgorithm expectedEncryptionAlgorithm, + List expectedPlaintexts + ) { + if (crossLanguageObjects.isEmpty()) { + throw new AssertionError("There is nothing to decrypt"); + } + + // Extract materials description from KeyMaterial + List metadata = (keyMaterial.getMaterialsDescription() != null) + ? metadataMapToList(keyMaterial.getMaterialsDescription()) + : new ArrayList<>(); + + List failures = new ArrayList<>(); + for (int i = 0; i < crossLanguageObjects.size(); i++) { + try { + String objectKey = crossLanguageObjects.get(i); + String expectedPlaintext = expectedPlaintexts.get(i); + + GetObjectOutput output = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .metadata(metadata) // Pass materials description for C++ + .build()); + + // Then: Pass + assertEquals(expectedPlaintext, new String(output.getBody().array())); + assertEquals( + expectedEncryptionAlgorithm, + GetEncryptionAlgorithm(objectKey), + "When decrypting the EncryptionAlgorithm does not match the expected value: " + expectedEncryptionAlgorithm + ); + } catch (Exception e) { + failures.add(String.format( + "Failed to decrypt object '%s' (index %d): %s - %s", + crossLanguageObjects.get(i), i, e.getClass().getSimpleName(), e.getMessage() + )); + } + } + + if (!failures.isEmpty()) { + throw new AssertionError(String.format( + "Decryption failed for %d out of %d objects:\n%s", + failures.size(), crossLanguageObjects.size(), + String.join("\n", failures) + )); + } + } + + public static void Decrypt_fails( + S3ECTestServerClient client, + String S3ECId, List crossLanguageObjects, + EncryptionAlgorithm expectedEncryptionAlgorithm + ) { + + if (crossLanguageObjects.isEmpty()) { + throw new AssertionError("There is nothing to decrypt"); + } + + List successfulDecrypt = new ArrayList<>(); + for (String objectKey : crossLanguageObjects) { + try { + + assertEquals( + expectedEncryptionAlgorithm, + GetEncryptionAlgorithm(objectKey), + "Before decrypting the EncryptionAlgorithm does not match the expected value: " + expectedEncryptionAlgorithm + ); + GetObjectOutput output = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + // It should fail to decrypt + successfulDecrypt.add(objectKey); + } catch (S3EncryptionClientError e) { + // This is a success + // TODO, add the failure message + } + } + + assertEquals(successfulDecrypt.size(), 0, "Decryption should have failed:" + String.join(",", successfulDecrypt)); + } + + /** + * Perform ranged get operation with specified byte range + */ + public static void RangedGet( + S3ECTestServerClient client, + String S3ECId, + List objectKeys, + long rangeStart, + long rangeEnd, + EncryptionAlgorithm expectedEncryptionAlgorithm + ) { + + if (objectKeys.isEmpty()) { + throw new AssertionError("There is nothing to get"); + } + + List failures = new ArrayList<>(); + for (String objectKey : objectKeys) { + try { + // Get the full object first to know expected content + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + byte[] fullContent = fullOutput.getBody().array(); + + // Perform ranged get + GetObjectOutput output = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .range("bytes=" + rangeStart + "-" + rangeEnd) + .build()); + + // Verify the ranged content matches expected slice + byte[] rangedContent = output.getBody().array(); + int startIndex = (int) rangeStart; + int endIndex = (int) Math.min(rangeEnd + 1, fullContent.length); // +1 because HTTP ranges are inclusive + byte[] expectedContent = Arrays.copyOfRange(fullContent, startIndex, endIndex); + assertArrayEquals(expectedContent, rangedContent, + "Ranged get returned unexpected data for:" + objectKey); + + // Verify encryption algorithm + assertEquals( + expectedEncryptionAlgorithm, + GetEncryptionAlgorithm(objectKey), + "Encryption algorithm mismatch for " + objectKey + ); + } catch (Exception e) { + failures.add(String.format( + "Failed ranged get on '%s': %s - %s", + objectKey, e.getClass().getSimpleName(), e.getMessage() + )); + } + } + + if (!failures.isEmpty()) { + throw new AssertionError(String.format( + "Ranged get failed for %d out of %d objects:\n%s", + failures.size(), objectKeys.size(), + String.join("\n", failures) + )); + } + } + + /** + * Perform ranged get operations that are expected to fail + */ + public static void RangedGet_fails( + S3ECTestServerClient client, + String S3ECId, + List objectKeys, + long rangeStart, + long rangeEnd, + EncryptionAlgorithm expectedEncryptionAlgorithm + ) { + + if (objectKeys.isEmpty()) { + throw new AssertionError("There is nothing to get"); + } + + List successfulGets = new ArrayList<>(); + for (String objectKey : objectKeys) { + try { + assertEquals( + expectedEncryptionAlgorithm, + GetEncryptionAlgorithm(objectKey), + "Encryption algorithm mismatch for " + objectKey + ); + + GetObjectOutput output = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .range("bytes=" + rangeStart + "-" + rangeEnd) + .build()); + + // Should have failed but didn't + successfulGets.add(objectKey); + } catch (S3EncryptionClientError e) { + // This is expected - the ranged get should fail + } + } + + assertEquals(0, successfulGets.size(), + "Ranged get should have failed for: " + String.join(", ", successfulGets)); + } +} diff --git a/test-server/java-v3-transition-server/.duvet/.gitignore b/test-server/java-v3-transition-server/.duvet/.gitignore new file mode 100644 index 00000000..93956e36 --- /dev/null +++ b/test-server/java-v3-transition-server/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ \ No newline at end of file diff --git a/test-server/java-v3-transition-server/.duvet/config.toml b/test-server/java-v3-transition-server/.duvet/config.toml new file mode 100644 index 00000000..645410cf --- /dev/null +++ b/test-server/java-v3-transition-server/.duvet/config.toml @@ -0,0 +1,27 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "s3ec-staging/**/*.java" + +# Include required specifications here +[[specification]] +source = "specification/s3-encryption/client.md" +[[specification]] +source = "specification/s3-encryption/decryption.md" +[[specification]] +source = "specification/s3-encryption/encryption.md" +[[specification]] +source = "specification/s3-encryption/key-commitment.md" +[[specification]] +source = "specification/s3-encryption/key-derivation.md" +[[specification]] +source = "specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "specification/s3-encryption/data-format/metadata-strategy.md" + +[report.html] +enabled = true + +# Enable snapshots to prevent requirement coverage regressions +[report.snapshot] +enabled = false diff --git a/test-server/java-v3-transition-server/.gitignore b/test-server/java-v3-transition-server/.gitignore new file mode 100644 index 00000000..e660fd93 --- /dev/null +++ b/test-server/java-v3-transition-server/.gitignore @@ -0,0 +1 @@ +bin/ diff --git a/test-server/java-v3-transition-server/Makefile b/test-server/java-v3-transition-server/Makefile new file mode 100644 index 00000000..81726b59 --- /dev/null +++ b/test-server/java-v3-transition-server/Makefile @@ -0,0 +1,42 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: build-server start-server stop-server wait-for-server + +PID_FILE := server.pid +PORT := 8094 + +build-server: + @echo "Building S3EC from source..." + cd s3ec-staging && mvn --batch-mode -no-transfer-progress clean compile && mvn -B -ntp install -DskipTests + @echo "S3EC build completed." + @echo "Building Java V3 Transition server..." + ./gradlew --build-cache --parallel --no-daemon build + +start-server: + @echo "Starting Java V3 Transition server..." + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + ./gradlew --build-cache --parallel run > server.log 2>&1 & echo $$! > $(PID_FILE) + @echo "Java V3 Transition server starting..." + +stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true + @if [ -f $(PID_FILE) ]; then \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ + fi + @rm -f server.log + @echo "Server stopped" + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT) + +duvet: + duvet report + +view-report-mac: + open .duvet/reports/report.html diff --git a/test-server/java-server/README.md b/test-server/java-v3-transition-server/README.md similarity index 63% rename from test-server/java-server/README.md rename to test-server/java-v3-transition-server/README.md index b2f5bb1b..5f08cc1c 100644 --- a/test-server/java-server/README.md +++ b/test-server/java-v3-transition-server/README.md @@ -1,6 +1,6 @@ -# S3EC Java Test Server +# S3EC Java V3 Test Server -This is the Java implementation of the S3ECTestServer framework. It provides a server implementation for testing S3 Encryption Client functionality. +This is the Java implementation of the S3ECTestServer framework for S3EC Java V3. It provides a server implementation for testing Java S3 Encryption Client V3 functionality. ## Overview @@ -18,6 +18,6 @@ To run the server: gradle run ``` -This will start the server running on port `8080`. +This will start the server running on port `8094`. The server is used as part of the testing framework to verify cross-language compatibility of the S3 Encryption Client implementations. diff --git a/test-server/java-server/build.gradle.kts b/test-server/java-v3-transition-server/build.gradle.kts similarity index 77% rename from test-server/java-server/build.gradle.kts rename to test-server/java-v3-transition-server/build.gradle.kts index ca793e56..7f474fa0 100644 --- a/test-server/java-server/build.gradle.kts +++ b/test-server/java-v3-transition-server/build.gradle.kts @@ -4,6 +4,10 @@ plugins { application } +// Dynamically read S3EC version from submodule's pom.xml +val s3ecVersion = file("s3ec-staging/pom.xml").readText() + .let { Regex("(.*?)").find(it)?.groupValues?.get(1) ?: "3.6.0" } + dependencies { val smithyJavaVersion: String by project @@ -13,8 +17,9 @@ dependencies { implementation("software.amazon.smithy.java:server-netty:$smithyJavaVersion") implementation("software.amazon.smithy.java:aws-server-restjson:$smithyJavaVersion") - compileOnly("software.amazon.awssdk:aws-sdk-java:2.31.66") - implementation("software.amazon.encryption.s3:amazon-s3-encryption-client-java:3.3.5") + // S3EC from local Maven repository (installed by mvn install) + // Version is dynamically read from s3ec-staging/pom.xml + implementation("software.amazon.encryption.s3:amazon-s3-encryption-client-java:$s3ecVersion") } // Use that application plugin to start the service via the `run` task. diff --git a/test-server/java-v3-transition-server/gradle.properties b/test-server/java-v3-transition-server/gradle.properties new file mode 100644 index 00000000..483cd315 --- /dev/null +++ b/test-server/java-v3-transition-server/gradle.properties @@ -0,0 +1,24 @@ +# Smithy versions +smithyJavaVersion=[0,1] +smithyGradleVersion=1.1.0 +smithyVersion=[1,2] + +# Performance optimization settings + +# Force no-daemon mode - ensures Gradle doesn't try to keep a daemon alive +org.gradle.daemon=false + +# Set minimal idle timeout for any daemon-like behavior (1 second) +org.gradle.daemon.idletimeout=1000 + +# JVM arguments to prevent forking a separate JVM process +# By matching the JVM args here with what Gradle expects, we avoid the +# "single-use Daemon process will be forked" behavior +org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseParallelGC + +# Keep builds fast with parallel execution and caching +org.gradle.parallel=true +org.gradle.caching=true + +# Configure on demand to reduce startup time +org.gradle.configureondemand=true diff --git a/test-server/java-server/gradle/wrapper/gradle-wrapper.jar b/test-server/java-v3-transition-server/gradle/wrapper/gradle-wrapper.jar similarity index 100% rename from test-server/java-server/gradle/wrapper/gradle-wrapper.jar rename to test-server/java-v3-transition-server/gradle/wrapper/gradle-wrapper.jar diff --git a/test-server/java-server/gradle/wrapper/gradle-wrapper.properties b/test-server/java-v3-transition-server/gradle/wrapper/gradle-wrapper.properties similarity index 100% rename from test-server/java-server/gradle/wrapper/gradle-wrapper.properties rename to test-server/java-v3-transition-server/gradle/wrapper/gradle-wrapper.properties diff --git a/test-server/java-server/gradlew b/test-server/java-v3-transition-server/gradlew similarity index 100% rename from test-server/java-server/gradlew rename to test-server/java-v3-transition-server/gradlew diff --git a/test-server/java-server/gradlew.bat b/test-server/java-v3-transition-server/gradlew.bat similarity index 96% rename from test-server/java-server/gradlew.bat rename to test-server/java-v3-transition-server/gradlew.bat index 7101f8e4..25da30db 100644 --- a/test-server/java-server/gradlew.bat +++ b/test-server/java-v3-transition-server/gradlew.bat @@ -1,92 +1,92 @@ -@rem -@rem Copyright 2015 the original author or authors. -@rem -@rem Licensed under the Apache License, Version 2.0 (the "License"); -@rem you may not use this file except in compliance with the License. -@rem You may obtain a copy of the License at -@rem -@rem https://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem - -@if "%DEBUG%"=="" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%"=="" set DIRNAME=. -@rem This is normally unused -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Resolve any "." and ".." in APP_HOME to make it shorter. -for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if %ERRORLEVEL% equ 0 goto execute - -echo. 1>&2 -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 -echo. 1>&2 -echo Please set the JAVA_HOME variable in your environment to match the 1>&2 -echo location of your Java installation. 1>&2 - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto execute - -echo. 1>&2 -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 -echo. 1>&2 -echo Please set the JAVA_HOME variable in your environment to match the 1>&2 -echo location of your Java installation. 1>&2 - -goto fail - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* - -:end -@rem End local scope for the variables with windows NT shell -if %ERRORLEVEL% equ 0 goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -set EXIT_CODE=%ERRORLEVEL% -if %EXIT_CODE% equ 0 set EXIT_CODE=1 -if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% -exit /b %EXIT_CODE% - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/test-server/java-server/license.txt b/test-server/java-v3-transition-server/license.txt similarity index 100% rename from test-server/java-server/license.txt rename to test-server/java-v3-transition-server/license.txt diff --git a/test-server/java-v3-transition-server/s3ec-staging b/test-server/java-v3-transition-server/s3ec-staging new file mode 160000 index 00000000..d829a235 --- /dev/null +++ b/test-server/java-v3-transition-server/s3ec-staging @@ -0,0 +1 @@ +Subproject commit d829a235854996e0f25736662510c2aa25e61fae diff --git a/test-server/java-server/settings.gradle.kts b/test-server/java-v3-transition-server/settings.gradle.kts similarity index 100% rename from test-server/java-server/settings.gradle.kts rename to test-server/java-v3-transition-server/settings.gradle.kts diff --git a/test-server/java-server/smithy-build.json b/test-server/java-v3-transition-server/smithy-build.json similarity index 100% rename from test-server/java-server/smithy-build.json rename to test-server/java-v3-transition-server/smithy-build.json diff --git a/test-server/java-v3-transition-server/specification b/test-server/java-v3-transition-server/specification new file mode 120000 index 00000000..b173f708 --- /dev/null +++ b/test-server/java-v3-transition-server/specification @@ -0,0 +1 @@ +../specification \ No newline at end of file diff --git a/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java new file mode 100644 index 00000000..956f454b --- /dev/null +++ b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java @@ -0,0 +1,198 @@ +package software.amazon.encryption.s3; + +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.encryption.s3.internal.InstructionFileConfig; +import software.amazon.encryption.s3.algorithms.AlgorithmSuite; +import software.amazon.encryption.s3.materials.AesKeyring; +import software.amazon.encryption.s3.materials.Keyring; +import software.amazon.encryption.s3.materials.KmsKeyring; +import software.amazon.encryption.s3.materials.PartialRsaKeyPair; +import software.amazon.encryption.s3.materials.RsaKeyring; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.GenericServerError; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.service.CreateClientOperation; +import software.amazon.smithy.java.server.RequestContext; + +import javax.crypto.spec.SecretKeySpec; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.security.KeyFactory; +import java.security.NoSuchAlgorithmException; +import java.security.PublicKey; +import java.security.interfaces.RSAPrivateCrtKey; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.RSAPublicKeySpec; +import java.security.spec.PKCS8EncodedKeySpec; +import java.util.Map; +import java.util.UUID; + +import static software.amazon.encryption.s3.CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT; +import static software.amazon.encryption.s3.CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT; +import static software.amazon.encryption.s3.CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT; + +public class CreateClientOperationImpl implements CreateClientOperation { + private final Map clientCache_; + private final Map keyringCache_; + + public CreateClientOperationImpl(Map clientCache, Map keyringCache) { + clientCache_ = clientCache; + keyringCache_ = keyringCache; + } + + // Copied from S3EC. + private boolean onlyOneNonNull(Object... values) { + boolean haveOneNonNull = false; + for (Object o : values) { + if (o != null) { + if (haveOneNonNull) { + return false; + } + + haveOneNonNull = true; + } + } + + return haveOneNonNull; + } + + @Override + public CreateClientOutput createClient(CreateClientInput input, RequestContext context) { + try { + KeyMaterial key = input.getConfig().getKeyMaterial(); + if (!onlyOneNonNull(key.getAesKey(), key.getKmsKeyId(), key.getRsaKey())) { + throw new RuntimeException("KeyMaterial must be only one, non-null input!"); + } + Keyring keyring; + if (key.getAesKey() != null) { + byte[] keyBytes = new byte[key.getAesKey().remaining()]; + key.getAesKey().get(keyBytes); + keyring = AesKeyring.builder() + .wrappingKey(new SecretKeySpec(keyBytes, "AES")) + .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) + .build(); + } else if (key.getRsaKey() != null) { + try { + byte[] keyBytes = new byte[key.getRsaKey().remaining()]; + key.getRsaKey().get(keyBytes); + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(keyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + RSAPrivateCrtKey privateKey = (RSAPrivateCrtKey) keyFactory.generatePrivate(keySpec); + RSAPublicKeySpec publicKeySpec = new RSAPublicKeySpec( + privateKey.getModulus(), + privateKey.getPublicExponent() + ); + + // Generate public key + PublicKey publicKey = keyFactory.generatePublic(publicKeySpec); + + keyring = RsaKeyring.builder() + .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) + .wrappingKeyPair(PartialRsaKeyPair.builder() + .publicKey(publicKey) + .privateKey(privateKey).build()) + .build(); + } catch (NoSuchAlgorithmException | InvalidKeySpecException nse) { + throw GenericServerError.builder() + .message(nse.getMessage()) + .build(); + } + } else if (key.getKmsKeyId() != null) { + keyring = KmsKeyring.builder() + .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) + .wrappingKeyId(key.getKmsKeyId()) + .build(); + } else { + throw new RuntimeException("No KeyMaterial found!"); + } + + // Configure S3 client with adaptive retry for throttling + RetryPolicy retryPolicy = RetryPolicy.builder() + .numRetries(5) + .throttlingBackoffStrategy(BackoffStrategy.defaultThrottlingStrategy()) + .build(); + + S3Client wrappedClient = S3Client.builder() + .overrideConfiguration(ClientOverrideConfiguration.builder() + .retryPolicy(retryPolicy) + .build()) + .build(); + + // V3 Transition server configuration + // Existing Builder defaults to FORBID_ENCRYPT and ALG_AES_256_GCM_IV12_TAG16_NO_KDF + S3EncryptionClient.Builder s3ClientBuilder = S3EncryptionClient.builder() + .wrappedClient(wrappedClient) + .keyring(keyring) + .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) + .enableLegacyUnauthenticatedModes(input.getConfig().isEnableLegacyUnauthenticatedModes()); + + // Instruction File Put Configuration + boolean instFilePut = false; + if (input.getConfig().getInstructionFileConfig() != null) { + instFilePut = input.getConfig().getInstructionFileConfig().isEnableInstructionFilePutObject(); + s3ClientBuilder.instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(S3Client.create()) + .enableInstructionFilePutObject(instFilePut) + .build()); + } + + // Configure commitment policy if provided + if (input.getConfig().getCommitmentPolicy() != null) { + CommitmentPolicy policy = getCommitmentPolicy(input.getConfig().getCommitmentPolicy()); + s3ClientBuilder.commitmentPolicy(policy); + } + + // Configure encryption algorithm if provided + if (input.getConfig().getEncryptionAlgorithm() != null) { + AlgorithmSuite algorithm = getAlgorithmSuite(input.getConfig().getEncryptionAlgorithm()); + s3ClientBuilder.encryptionAlgorithm(algorithm); + } + + S3Client s3Client = s3ClientBuilder.build(); + + UUID uuid = UUID.randomUUID(); + String uuidString = uuid.toString(); + clientCache_.put(uuidString, s3Client); + keyringCache_.put(uuidString, keyring); + return CreateClientOutput.builder() + .clientId(uuidString) + .build(); + } catch (Exception e) { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw GenericServerError.builder() + .message(stackTrace) + .build(); + } + } + + private static AlgorithmSuite getAlgorithmSuite(EncryptionAlgorithm input) { + if (input.equals(EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF)) { + return AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF; + } else if (input.equals(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF)) { + return AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF; + } else if (input.equals(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY)) { + return AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY; + } else { + throw new RuntimeException("Unknown encryption algorithm: " + input); + } + } + + private static software.amazon.encryption.s3.CommitmentPolicy getCommitmentPolicy(software.amazon.encryption.s3.model.CommitmentPolicy input) { + if (input.equals(software.amazon.encryption.s3.model.CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT)) { + return FORBID_ENCRYPT_ALLOW_DECRYPT; + } else if (input.equals(software.amazon.encryption.s3.model.CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT)) { + return REQUIRE_ENCRYPT_ALLOW_DECRYPT; + } else if (input.equals(software.amazon.encryption.s3.model.CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT)) { + return REQUIRE_ENCRYPT_REQUIRE_DECRYPT; + } else { + throw new RuntimeException("Unknown commitment policy: " + input); + } + } +} diff --git a/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java new file mode 100644 index 00000000..d3ab8289 --- /dev/null +++ b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java @@ -0,0 +1,88 @@ +package software.amazon.encryption.s3; + +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.encryption.s3.S3EncryptionClientException; +import software.amazon.smithy.java.server.RequestContext; +import software.amazon.encryption.s3.model.GenericServerError; +import software.amazon.encryption.s3.model.GetObjectInput; +import software.amazon.encryption.s3.model.GetObjectOutput; +import software.amazon.encryption.s3.model.S3EncryptionClientError; +import software.amazon.encryption.s3.service.GetObjectOperation; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static software.amazon.encryption.s3.S3EncryptionClient.withAdditionalConfiguration; +import static software.amazon.encryption.s3.MetadataUtils.metadataListToMap; +import static software.amazon.encryption.s3.MetadataUtils.metadataMapToList; + +public class GetObjectOperationImpl implements GetObjectOperation { + private Map clientCache_; + + public GetObjectOperationImpl(Map clientCache) { + clientCache_ = clientCache; + } + + @Override + public GetObjectOutput getObject(GetObjectInput input, RequestContext context) { + try { + S3Client s3Client = clientCache_.get(input.getClientID()); + Map ecMap = metadataListToMap(input.getMetadata()); + + try { + ResponseBytes resp = s3Client.getObjectAsBytes(builder -> { + builder.bucket(input.getBucket()) + .key(input.getKey()); + + // Add custom instruction file suffix if provided + if (input.getInstructionFileSuffix() != null && !input.getInstructionFileSuffix().isEmpty()) { + builder.overrideConfiguration(config -> config + .putExecutionAttribute(S3EncryptionClient.CUSTOM_INSTRUCTION_FILE_SUFFIX, input.getInstructionFileSuffix()) + .applyMutation(c -> withAdditionalConfiguration(ecMap).accept(c))); + } else { + builder.overrideConfiguration(withAdditionalConfiguration(ecMap)); + } + + // Add range header if provided + if (input.getRange() != null && !input.getRange().isEmpty()) { + builder.range(input.getRange()); + } + }); + + List mdAsList = metadataMapToList(resp.response().metadata()); + // Can't use asBB else it gets mad bc cant access backing array + ByteBuffer bb = ByteBuffer.wrap(resp.asByteArray()); + GetObjectOutput output = GetObjectOutput.builder() + .body(bb) + .metadata(mdAsList) + .build(); + return output; + } catch (S3EncryptionClientException s3EncryptionClientException) { + // Modeled exceptions MUST be returned as such + StringWriter sw = new StringWriter(); + s3EncryptionClientException.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw S3EncryptionClientError.builder() + .message(stackTrace) + .build(); + } + } catch (Exception e) { + // Don't wrap modeled errors + if (e instanceof S3EncryptionClientError) { + throw e; + } + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw GenericServerError.builder() + .message(stackTrace) + .build(); + } + } +} diff --git a/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java new file mode 100644 index 00000000..9eba6a3d --- /dev/null +++ b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java @@ -0,0 +1,43 @@ +package software.amazon.encryption.s3; + +import software.amazon.encryption.s3.model.GenericServerError; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class MetadataUtils { + + /** + * Annoyingly, Smithy doesn't provide an interface for map types + * in HTTP headers, so we have to do the serde ourselves + */ + public static List metadataMapToList(Map md) { + List mdAsList = new ArrayList<>(md.size()); + for (Map.Entry keyValue : md.entrySet()) { + mdAsList.add("[" + keyValue.getKey() + "]:[" + keyValue.getValue() + "]"); + } + return mdAsList; + } + + public static Map metadataListToMap(List mdList) { + Map md = new HashMap<>(); + for (String entry : mdList) { + // Split on "]:[" to separate key and value + String[] parts = entry.split("]:\\["); + if (parts.length == 2) { + // Remove remaining brackets from start and end + String key = parts[0].substring(1); + String value = parts[1].substring(0, parts[1].length() - 1); + md.put(key, value); + } else { + throw GenericServerError.builder() + .message("Malformed metadata list entry: " + entry) + .build(); + } + } + return md; + } + +} diff --git a/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java new file mode 100644 index 00000000..ca76e83f --- /dev/null +++ b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java @@ -0,0 +1,55 @@ +package software.amazon.encryption.s3; + +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.smithy.java.server.RequestContext; +import software.amazon.encryption.s3.model.GenericServerError; +import software.amazon.encryption.s3.model.PutObjectInput; +import software.amazon.encryption.s3.model.PutObjectOutput; +import software.amazon.encryption.s3.service.PutObjectOperation; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Map; +import java.util.stream.Collectors; + +import static software.amazon.encryption.s3.S3EncryptionClient.withAdditionalConfiguration; +import static software.amazon.encryption.s3.MetadataUtils.metadataListToMap; + +public class PutObjectOperationImpl implements PutObjectOperation { + + private Map clientCache_; + + public PutObjectOperationImpl(Map clientCache) { + clientCache_ = clientCache; + } + + @Override + public PutObjectOutput putObject(PutObjectInput input, RequestContext context) { + try { + final Map metadata = metadataListToMap(input.getMetadata()); + S3Client s3Client = clientCache_.get(input.getClientID()); + s3Client.putObject(builder -> builder + .bucket(input.getBucket()) + .key(input.getKey()) + .overrideConfiguration(withAdditionalConfiguration(metadata)), + RequestBody.fromByteBuffer(input.getBody()) + ); + // The real S3 doesn't provide bucket/key/metadata, so Test doesn't need to either, but we do anyway + return PutObjectOutput.builder() + .bucket(input.getBucket()) + .key(input.getKey()) + .metadata(input.getMetadata()) + .build(); + } catch (Exception e) { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw GenericServerError.builder() + .message(stackTrace) + .build(); + } + } +} diff --git a/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/ReEncryptOperationImpl.java b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/ReEncryptOperationImpl.java new file mode 100644 index 00000000..7a809761 --- /dev/null +++ b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/ReEncryptOperationImpl.java @@ -0,0 +1,183 @@ +package software.amazon.encryption.s3; + +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.encryption.s3.internal.ReEncryptInstructionFileRequest; +import software.amazon.encryption.s3.internal.ReEncryptInstructionFileResponse; +import software.amazon.encryption.s3.materials.AesKeyring; +import software.amazon.encryption.s3.materials.MaterialsDescription; +import software.amazon.encryption.s3.materials.PartialRsaKeyPair; +import software.amazon.encryption.s3.materials.RawKeyring; +import software.amazon.encryption.s3.materials.RsaKeyring; +import software.amazon.encryption.s3.model.GenericServerError; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.ReEncryptInput; +import software.amazon.encryption.s3.model.ReEncryptOutput; +import software.amazon.encryption.s3.model.S3EncryptionClientError; +import software.amazon.encryption.s3.service.ReEncryptOperation; +import software.amazon.smithy.java.server.RequestContext; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.security.KeyFactory; +import java.security.PublicKey; +import java.security.interfaces.RSAPrivateCrtKey; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.RSAPublicKeySpec; +import java.util.Map; + +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; + +public class ReEncryptOperationImpl implements ReEncryptOperation { + private final Map clientCache_; + + public ReEncryptOperationImpl(Map clientCache) { + clientCache_ = clientCache; + } + + @Override + public ReEncryptOutput reEncrypt(ReEncryptInput input, RequestContext context) { + try { + S3Client s3Client = clientCache_.get(input.getClientID()); + + // Ensure we have an S3EncryptionClient, not just a plain S3Client + if (!(s3Client instanceof S3EncryptionClient)) { + throw new IllegalStateException( + "Client " + input.getClientID() + " is not an S3EncryptionClient"); + } + + S3EncryptionClient s3EncryptionClient = (S3EncryptionClient) s3Client; + + // Create a new keyring from the provided newKeyMaterial + KeyMaterial newKeyMaterial = input.getNewKeyMaterial(); + if (newKeyMaterial == null) { + throw new IllegalStateException( + "newKeyMaterial is required for ReEncrypt operation"); + } + + RawKeyring newKeyring = createKeyringFromMaterial(newKeyMaterial); + + try { + // Build the ReEncryptInstructionFileRequest + ReEncryptInstructionFileRequest.Builder requestBuilder = + ReEncryptInstructionFileRequest.builder() + .bucket(input.getBucket()) + .key(input.getKey()) + .newKeyring(newKeyring); + + // Add optional instruction file suffix if provided + if (input.getInstructionFileSuffix() != null && !input.getInstructionFileSuffix().isEmpty()) { + requestBuilder.instructionFileSuffix(input.getInstructionFileSuffix()); + } + + // Add optional enforceRotation if provided + if (input.isEnforceRotation() != null) { + requestBuilder.enforceRotation(input.isEnforceRotation()); + } + + ReEncryptInstructionFileRequest reEncryptRequest = requestBuilder.build(); + + // Perform the re-encryption + ReEncryptInstructionFileResponse response = + s3EncryptionClient.reEncryptInstructionFile(reEncryptRequest); + + // Build and return the output + return ReEncryptOutput.builder() + .bucket(response.bucket()) + .key(response.key()) + .instructionFileSuffix(response.instructionFileSuffix()) + .enforceRotation(response.enforceRotation()) + .build(); + + } catch (S3EncryptionClientException s3EncryptionClientException) { + // Modeled exceptions MUST be returned as such + StringWriter sw = new StringWriter(); + s3EncryptionClientException.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw S3EncryptionClientError.builder() + .message(stackTrace) + .build(); + } + } catch (Exception e) { + // Don't wrap modeled errors + if (e instanceof S3EncryptionClientError) { + throw e; + } + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw GenericServerError.builder() + .message(stackTrace) + .build(); + } + } + + /** + * Creates a RawKeyring from KeyMaterial. + * The KeyMaterial should have exactly one of: aesKey, rsaKey, or kmsKeyId set. + */ + private RawKeyring createKeyringFromMaterial(KeyMaterial keyMaterial) { + try { + // Get materials description from KeyMaterial if provided + MaterialsDescription materialsDescription = null; + if (keyMaterial.getMaterialsDescription() != null && !keyMaterial.getMaterialsDescription().isEmpty()) { + MaterialsDescription.Builder builder = MaterialsDescription.builder(); + for (Map.Entry entry : keyMaterial.getMaterialsDescription().entrySet()) { + builder.put(entry.getKey(), entry.getValue()); + } + materialsDescription = builder.build(); + } + + // Check for AES key + if (keyMaterial.getAesKey() != null) { + byte[] aesKeyBytes = new byte[keyMaterial.getAesKey().remaining()]; + keyMaterial.getAesKey().get(aesKeyBytes); + SecretKey secretKey = new SecretKeySpec(aesKeyBytes, "AES"); + + AesKeyring.Builder keyringBuilder = AesKeyring.builder() + .wrappingKey(secretKey); + + if (materialsDescription != null) { + keyringBuilder.materialsDescription(materialsDescription); + } + + return keyringBuilder.build(); + } + + // Check for RSA key + if (keyMaterial.getRsaKey() != null) { + byte[] rsaKeyBytes = new byte[keyMaterial.getRsaKey().remaining()]; + keyMaterial.getRsaKey().get(rsaKeyBytes); + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(rsaKeyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + RSAPrivateCrtKey privateKey = (RSAPrivateCrtKey) keyFactory.generatePrivate(keySpec); + + // Derive the public key from the private key + RSAPublicKeySpec publicKeySpec = new RSAPublicKeySpec( + privateKey.getModulus(), + privateKey.getPublicExponent() + ); + PublicKey publicKey = keyFactory.generatePublic(publicKeySpec); + + PartialRsaKeyPair keyPair = PartialRsaKeyPair.builder() + .privateKey(privateKey) + .publicKey(publicKey) + .build(); + + RsaKeyring.Builder keyringBuilder = RsaKeyring.builder() + .wrappingKeyPair(keyPair); + + if (materialsDescription != null) { + keyringBuilder.materialsDescription(materialsDescription); + } + + return keyringBuilder.build(); + } + + throw new IllegalStateException( + "KeyMaterial must have either aesKey or rsaKey set"); + } catch (Exception e) { + throw new IllegalStateException("Failed to create keyring from KeyMaterial: " + e.getMessage(), e); + } + } +} diff --git a/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java new file mode 100644 index 00000000..78c84dff --- /dev/null +++ b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java @@ -0,0 +1,56 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.encryption.s3; + +import software.amazon.awssdk.services.s3.S3Client; + +import java.net.URI; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; + +import software.amazon.smithy.java.server.Server; +import software.amazon.encryption.s3.service.S3ECTestServer; + +public class S3ECJavaTestServer implements Runnable { + static final URI endpoint = URI.create("http://localhost:8094"); + + public static void main(String[] args) { + new S3ECJavaTestServer().run(); + } + + @Override + public void run() { + // All the S3EC instances live here. + // Obviously this can get messy in a real service. + // Assume that the tests behave and don't induce weird race conditions. + Map clientCache = new ConcurrentHashMap<>(); + Map keyringCache = new ConcurrentHashMap<>(); + + Server server = Server.builder() + .endpoints(endpoint) + .addService( + S3ECTestServer.builder() + .addCreateClientOperation(new CreateClientOperationImpl(clientCache, keyringCache)) + .addGetObjectOperation(new GetObjectOperationImpl(clientCache)) + .addPutObjectOperation(new PutObjectOperationImpl(clientCache)) + .addReEncryptOperation(new ReEncryptOperationImpl(clientCache)) + .build()) + .build(); + System.out.println("Starting server..."); + server.start(); + try { + Thread.currentThread().join(); + } catch (InterruptedException e) { + System.out.println("Stopping server..."); + try { + server.shutdown().get(); + } catch (InterruptedException | ExecutionException ex) { + throw new RuntimeException(ex); + } + } + } +} diff --git a/test-server/java-v4-server/.duvet/.gitignore b/test-server/java-v4-server/.duvet/.gitignore new file mode 100644 index 00000000..93956e36 --- /dev/null +++ b/test-server/java-v4-server/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ \ No newline at end of file diff --git a/test-server/java-v4-server/.duvet/config.toml b/test-server/java-v4-server/.duvet/config.toml new file mode 100644 index 00000000..645410cf --- /dev/null +++ b/test-server/java-v4-server/.duvet/config.toml @@ -0,0 +1,27 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "s3ec-staging/**/*.java" + +# Include required specifications here +[[specification]] +source = "specification/s3-encryption/client.md" +[[specification]] +source = "specification/s3-encryption/decryption.md" +[[specification]] +source = "specification/s3-encryption/encryption.md" +[[specification]] +source = "specification/s3-encryption/key-commitment.md" +[[specification]] +source = "specification/s3-encryption/key-derivation.md" +[[specification]] +source = "specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "specification/s3-encryption/data-format/metadata-strategy.md" + +[report.html] +enabled = true + +# Enable snapshots to prevent requirement coverage regressions +[report.snapshot] +enabled = false diff --git a/test-server/java-v4-server/.gitignore b/test-server/java-v4-server/.gitignore new file mode 100644 index 00000000..e660fd93 --- /dev/null +++ b/test-server/java-v4-server/.gitignore @@ -0,0 +1 @@ +bin/ diff --git a/test-server/java-v4-server/Makefile b/test-server/java-v4-server/Makefile new file mode 100644 index 00000000..3d1aae2a --- /dev/null +++ b/test-server/java-v4-server/Makefile @@ -0,0 +1,42 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: build-server start-server stop-server wait-for-server + +PID_FILE := server.pid +PORT := 8088 + +build-server: + @echo "Building S3EC from source..." + cd s3ec-staging && mvn --batch-mode -no-transfer-progress clean compile && mvn -B -ntp install -DskipTests + @echo "S3EC build completed." + @echo "Building Java V4 server..." + ./gradlew --build-cache --parallel --no-daemon build + +start-server: + @echo "Starting Java V4 server..." + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + ./gradlew --build-cache --parallel run > server.log 2>&1 & echo $$! > $(PID_FILE) + @echo "Java V4 server starting..." + +stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true + @if [ -f $(PID_FILE) ]; then \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ + fi + @rm -f server.log + @echo "Server stopped" + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT) + +duvet: + duvet report + +view-report-mac: + open .duvet/reports/report.html diff --git a/test-server/java-v4-server/README.md b/test-server/java-v4-server/README.md new file mode 100644 index 00000000..70d60914 --- /dev/null +++ b/test-server/java-v4-server/README.md @@ -0,0 +1,23 @@ +# S3EC Java V4 (Improved) Test Server + +This is the Java implementation of the S3ECTestServer framework for S3EC Java V4 (Improved). It provides a server implementation for testing Java S3 Encryption Client V4 (Improved) functionality. + +## Overview + +The S3ECJavaTestServer implements the S3ECTestServer service defined in the shared Smithy model. It provides endpoints for: + +- Creating S3 Encryption Clients +- Putting objects with encryption +- Getting and decrypting objects + +## Usage + +To run the server: + +```console +gradle run +``` + +This will start the server running on port `8088`. + +The server is used as part of the testing framework to verify cross-language compatibility of the S3 Encryption Client implementations. diff --git a/test-server/java-v4-server/build.gradle.kts b/test-server/java-v4-server/build.gradle.kts new file mode 100644 index 00000000..d55d93d7 --- /dev/null +++ b/test-server/java-v4-server/build.gradle.kts @@ -0,0 +1,60 @@ +plugins { + `java-library` + id("software.amazon.smithy.gradle.smithy-base") + application +} + +// Dynamically read S3EC version from submodule's pom.xml +val s3ecVersion = file("s3ec-staging/pom.xml").readText() + .let { Regex("(.*?)").find(it)?.groupValues?.get(1) ?: "4.0.0" } + +dependencies { + val smithyJavaVersion: String by project + + smithyBuild("software.amazon.smithy.java:plugins:$smithyJavaVersion") + + implementation("software.amazon.smithy:smithy-rules-engine:1.59.0") + implementation("software.amazon.smithy.java:server-netty:$smithyJavaVersion") + implementation("software.amazon.smithy.java:aws-server-restjson:$smithyJavaVersion") + + // S3EC from local Maven repository (installed by mvn install) + // Version is dynamically read from s3ec-staging/pom.xml + implementation("software.amazon.encryption.s3:amazon-s3-encryption-client-java:$s3ecVersion") +} + +// Use that application plugin to start the service via the `run` task. +application { + mainClass = "software.amazon.encryption.s3.S3ECJavaTestServer" +} + +// Add generated Java files to the main sourceSet +afterEvaluate { + val serverPath = smithy.getPluginProjectionPath(smithy.sourceProjection.get(), "java-server-codegen") + sourceSets { + main { + java { + srcDir(serverPath) + } + } + } +} + +tasks { + compileJava { + dependsOn(smithyBuild) + } +} + +// Helps Intellij IDE's discover smithy models +sourceSets { + main { + java { + srcDir("../model") + } + } +} + +repositories { + mavenLocal() + mavenCentral() +} diff --git a/test-server/java-v4-server/gradle.properties b/test-server/java-v4-server/gradle.properties new file mode 100644 index 00000000..483cd315 --- /dev/null +++ b/test-server/java-v4-server/gradle.properties @@ -0,0 +1,24 @@ +# Smithy versions +smithyJavaVersion=[0,1] +smithyGradleVersion=1.1.0 +smithyVersion=[1,2] + +# Performance optimization settings + +# Force no-daemon mode - ensures Gradle doesn't try to keep a daemon alive +org.gradle.daemon=false + +# Set minimal idle timeout for any daemon-like behavior (1 second) +org.gradle.daemon.idletimeout=1000 + +# JVM arguments to prevent forking a separate JVM process +# By matching the JVM args here with what Gradle expects, we avoid the +# "single-use Daemon process will be forked" behavior +org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseParallelGC + +# Keep builds fast with parallel execution and caching +org.gradle.parallel=true +org.gradle.caching=true + +# Configure on demand to reduce startup time +org.gradle.configureondemand=true diff --git a/test-server/java-v4-server/gradle/wrapper/gradle-wrapper.jar b/test-server/java-v4-server/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..e6441136f3d4ba8a0da8d277868979cfbc8ad796 GIT binary patch literal 43453 zcma&N1CXTcmMvW9vTb(Rwr$&4wr$(C?dmSu>@vG-+vuvg^_??!{yS%8zW-#zn-LkA z5&1^$^{lnmUON?}LBF8_K|(?T0Ra(xUH{($5eN!MR#ZihR#HxkUPe+_R8Cn`RRs(P z_^*#_XlXmGv7!4;*Y%p4nw?{bNp@UZHv1?Um8r6)Fei3p@ClJn0ECfg1hkeuUU@Or zDaPa;U3fE=3L}DooL;8f;P0ipPt0Z~9P0)lbStMS)ag54=uL9ia-Lm3nh|@(Y?B`; zx_#arJIpXH!U{fbCbI^17}6Ri*H<>OLR%c|^mh8+)*h~K8Z!9)DPf zR2h?lbDZQ`p9P;&DQ4F0sur@TMa!Y}S8irn(%d-gi0*WxxCSk*A?3lGh=gcYN?FGl z7D=Js!i~0=u3rox^eO3i@$0=n{K1lPNU zwmfjRVmLOCRfe=seV&P*1Iq=^i`502keY8Uy-WNPwVNNtJFx?IwAyRPZo2Wo1+S(xF37LJZ~%i)kpFQ3Fw=mXfd@>%+)RpYQLnr}B~~zoof(JVm^^&f zxKV^+3D3$A1G;qh4gPVjhrC8e(VYUHv#dy^)(RoUFM?o%W-EHxufuWf(l*@-l+7vt z=l`qmR56K~F|v<^Pd*p~1_y^P0P^aPC##d8+HqX4IR1gu+7w#~TBFphJxF)T$2WEa zxa?H&6=Qe7d(#tha?_1uQys2KtHQ{)Qco)qwGjrdNL7thd^G5i8Os)CHqc>iOidS} z%nFEDdm=GXBw=yXe1W-ShHHFb?Cc70+$W~z_+}nAoHFYI1MV1wZegw*0y^tC*s%3h zhD3tN8b=Gv&rj}!SUM6|ajSPp*58KR7MPpI{oAJCtY~JECm)*m_x>AZEu>DFgUcby z1Qaw8lU4jZpQ_$;*7RME+gq1KySGG#Wql>aL~k9tLrSO()LWn*q&YxHEuzmwd1?aAtI zBJ>P=&$=l1efe1CDU;`Fd+_;&wI07?V0aAIgc(!{a z0Jg6Y=inXc3^n!U0Atk`iCFIQooHqcWhO(qrieUOW8X(x?(RD}iYDLMjSwffH2~tB z)oDgNBLB^AJBM1M^c5HdRx6fBfka`(LD-qrlh5jqH~);#nw|iyp)()xVYak3;Ybik z0j`(+69aK*B>)e_p%=wu8XC&9e{AO4c~O1U`5X9}?0mrd*m$_EUek{R?DNSh(=br# z#Q61gBzEpmy`$pA*6!87 zSDD+=@fTY7<4A?GLqpA?Pb2z$pbCc4B4zL{BeZ?F-8`s$?>*lXXtn*NC61>|*w7J* z$?!iB{6R-0=KFmyp1nnEmLsA-H0a6l+1uaH^g%c(p{iT&YFrbQ$&PRb8Up#X3@Zsk zD^^&LK~111%cqlP%!_gFNa^dTYT?rhkGl}5=fL{a`UViaXWI$k-UcHJwmaH1s=S$4 z%4)PdWJX;hh5UoK?6aWoyLxX&NhNRqKam7tcOkLh{%j3K^4Mgx1@i|Pi&}<^5>hs5 zm8?uOS>%)NzT(%PjVPGa?X%`N2TQCKbeH2l;cTnHiHppPSJ<7y-yEIiC!P*ikl&!B z%+?>VttCOQM@ShFguHVjxX^?mHX^hSaO_;pnyh^v9EumqSZTi+#f&_Vaija0Q-e*| z7ulQj6Fs*bbmsWp{`auM04gGwsYYdNNZcg|ph0OgD>7O}Asn7^Z=eI>`$2*v78;sj-}oMoEj&@)9+ycEOo92xSyY344^ z11Hb8^kdOvbf^GNAK++bYioknrpdN>+u8R?JxG=!2Kd9r=YWCOJYXYuM0cOq^FhEd zBg2puKy__7VT3-r*dG4c62Wgxi52EMCQ`bKgf*#*ou(D4-ZN$+mg&7$u!! z-^+Z%;-3IDwqZ|K=ah85OLwkO zKxNBh+4QHh)u9D?MFtpbl)us}9+V!D%w9jfAMYEb>%$A;u)rrI zuBudh;5PN}_6J_}l55P3l_)&RMlH{m!)ai-i$g)&*M`eN$XQMw{v^r@-125^RRCF0 z^2>|DxhQw(mtNEI2Kj(;KblC7x=JlK$@78`O~>V!`|1Lm-^JR$-5pUANAnb(5}B}JGjBsliK4& zk6y(;$e&h)lh2)L=bvZKbvh@>vLlreBdH8No2>$#%_Wp1U0N7Ank!6$dFSi#xzh|( zRi{Uw%-4W!{IXZ)fWx@XX6;&(m_F%c6~X8hx=BN1&q}*( zoaNjWabE{oUPb!Bt$eyd#$5j9rItB-h*5JiNi(v^e|XKAj*8(k<5-2$&ZBR5fF|JA z9&m4fbzNQnAU}r8ab>fFV%J0z5awe#UZ|bz?Ur)U9bCIKWEzi2%A+5CLqh?}K4JHi z4vtM;+uPsVz{Lfr;78W78gC;z*yTch~4YkLr&m-7%-xc ztw6Mh2d>_iO*$Rd8(-Cr1_V8EO1f*^@wRoSozS) zy1UoC@pruAaC8Z_7~_w4Q6n*&B0AjOmMWa;sIav&gu z|J5&|{=a@vR!~k-OjKEgPFCzcJ>#A1uL&7xTDn;{XBdeM}V=l3B8fE1--DHjSaxoSjNKEM9|U9#m2<3>n{Iuo`r3UZp;>GkT2YBNAh|b z^jTq-hJp(ebZh#Lk8hVBP%qXwv-@vbvoREX$TqRGTgEi$%_F9tZES@z8Bx}$#5eeG zk^UsLBH{bc2VBW)*EdS({yw=?qmevwi?BL6*=12k9zM5gJv1>y#ML4!)iiPzVaH9% zgSImetD@dam~e>{LvVh!phhzpW+iFvWpGT#CVE5TQ40n%F|p(sP5mXxna+Ev7PDwA zamaV4m*^~*xV+&p;W749xhb_X=$|LD;FHuB&JL5?*Y2-oIT(wYY2;73<^#46S~Gx| z^cez%V7x$81}UWqS13Gz80379Rj;6~WdiXWOSsdmzY39L;Hg3MH43o*y8ibNBBH`(av4|u;YPq%{R;IuYow<+GEsf@R?=@tT@!}?#>zIIn0CoyV!hq3mw zHj>OOjfJM3F{RG#6ujzo?y32m^tgSXf@v=J$ELdJ+=5j|=F-~hP$G&}tDZsZE?5rX ztGj`!S>)CFmdkccxM9eGIcGnS2AfK#gXwj%esuIBNJQP1WV~b~+D7PJTmWGTSDrR` zEAu4B8l>NPuhsk5a`rReSya2nfV1EK01+G!x8aBdTs3Io$u5!6n6KX%uv@DxAp3F@{4UYg4SWJtQ-W~0MDb|j-$lwVn znAm*Pl!?Ps&3wO=R115RWKb*JKoexo*)uhhHBncEDMSVa_PyA>k{Zm2(wMQ(5NM3# z)jkza|GoWEQo4^s*wE(gHz?Xsg4`}HUAcs42cM1-qq_=+=!Gk^y710j=66(cSWqUe zklbm8+zB_syQv5A2rj!Vbw8;|$@C!vfNmNV!yJIWDQ>{+2x zKjuFX`~~HKG~^6h5FntRpnnHt=D&rq0>IJ9#F0eM)Y-)GpRjiN7gkA8wvnG#K=q{q z9dBn8_~wm4J<3J_vl|9H{7q6u2A!cW{bp#r*-f{gOV^e=8S{nc1DxMHFwuM$;aVI^ zz6A*}m8N-&x8;aunp1w7_vtB*pa+OYBw=TMc6QK=mbA-|Cf* zvyh8D4LRJImooUaSb7t*fVfih<97Gf@VE0|z>NcBwBQze);Rh!k3K_sfunToZY;f2 z^HmC4KjHRVg+eKYj;PRN^|E0>Gj_zagfRbrki68I^#~6-HaHg3BUW%+clM1xQEdPYt_g<2K+z!$>*$9nQ>; zf9Bei{?zY^-e{q_*|W#2rJG`2fy@{%6u0i_VEWTq$*(ZN37|8lFFFt)nCG({r!q#9 z5VK_kkSJ3?zOH)OezMT{!YkCuSSn!K#-Rhl$uUM(bq*jY? zi1xbMVthJ`E>d>(f3)~fozjg^@eheMF6<)I`oeJYx4*+M&%c9VArn(OM-wp%M<-`x z7sLP1&3^%Nld9Dhm@$3f2}87!quhI@nwd@3~fZl_3LYW-B?Ia>ui`ELg z&Qfe!7m6ze=mZ`Ia9$z|ARSw|IdMpooY4YiPN8K z4B(ts3p%2i(Td=tgEHX z0UQ_>URBtG+-?0E;E7Ld^dyZ;jjw0}XZ(}-QzC6+NN=40oDb2^v!L1g9xRvE#@IBR zO!b-2N7wVfLV;mhEaXQ9XAU+>=XVA6f&T4Z-@AX!leJ8obP^P^wP0aICND?~w&NykJ#54x3_@r7IDMdRNy4Hh;h*!u(Ol(#0bJdwEo$5437-UBjQ+j=Ic>Q2z` zJNDf0yO6@mr6y1#n3)s(W|$iE_i8r@Gd@!DWDqZ7J&~gAm1#~maIGJ1sls^gxL9LLG_NhU!pTGty!TbhzQnu)I*S^54U6Yu%ZeCg`R>Q zhBv$n5j0v%O_j{QYWG!R9W?5_b&67KB$t}&e2LdMvd(PxN6Ir!H4>PNlerpBL>Zvyy!yw z-SOo8caEpDt(}|gKPBd$qND5#a5nju^O>V&;f890?yEOfkSG^HQVmEbM3Ugzu+UtH zC(INPDdraBN?P%kE;*Ae%Wto&sgw(crfZ#Qy(<4nk;S|hD3j{IQRI6Yq|f^basLY; z-HB&Je%Gg}Jt@={_C{L$!RM;$$|iD6vu#3w?v?*;&()uB|I-XqEKqZPS!reW9JkLewLb!70T7n`i!gNtb1%vN- zySZj{8-1>6E%H&=V}LM#xmt`J3XQoaD|@XygXjdZ1+P77-=;=eYpoEQ01B@L*a(uW zrZeZz?HJsw_4g0vhUgkg@VF8<-X$B8pOqCuWAl28uB|@r`19DTUQQsb^pfqB6QtiT z*`_UZ`fT}vtUY#%sq2{rchyfu*pCg;uec2$-$N_xgjZcoumE5vSI{+s@iLWoz^Mf; zuI8kDP{!XY6OP~q5}%1&L}CtfH^N<3o4L@J@zg1-mt{9L`s^z$Vgb|mr{@WiwAqKg zp#t-lhrU>F8o0s1q_9y`gQNf~Vb!F%70f}$>i7o4ho$`uciNf=xgJ>&!gSt0g;M>*x4-`U)ysFW&Vs^Vk6m%?iuWU+o&m(2Jm26Y(3%TL; zA7T)BP{WS!&xmxNw%J=$MPfn(9*^*TV;$JwRy8Zl*yUZi8jWYF>==j~&S|Xinsb%c z2?B+kpet*muEW7@AzjBA^wAJBY8i|#C{WtO_or&Nj2{=6JTTX05}|H>N2B|Wf!*3_ z7hW*j6p3TvpghEc6-wufFiY!%-GvOx*bZrhZu+7?iSrZL5q9}igiF^*R3%DE4aCHZ zqu>xS8LkW+Auv%z-<1Xs92u23R$nk@Pk}MU5!gT|c7vGlEA%G^2th&Q*zfg%-D^=f z&J_}jskj|Q;73NP4<4k*Y%pXPU2Thoqr+5uH1yEYM|VtBPW6lXaetokD0u z9qVek6Q&wk)tFbQ8(^HGf3Wp16gKmr>G;#G(HRBx?F`9AIRboK+;OfHaLJ(P>IP0w zyTbTkx_THEOs%Q&aPrxbZrJlio+hCC_HK<4%f3ZoSAyG7Dn`=X=&h@m*|UYO-4Hq0 z-Bq&+Ie!S##4A6OGoC~>ZW`Y5J)*ouaFl_e9GA*VSL!O_@xGiBw!AF}1{tB)z(w%c zS1Hmrb9OC8>0a_$BzeiN?rkPLc9%&;1CZW*4}CDDNr2gcl_3z+WC15&H1Zc2{o~i) z)LLW=WQ{?ricmC`G1GfJ0Yp4Dy~Ba;j6ZV4r{8xRs`13{dD!xXmr^Aga|C=iSmor% z8hi|pTXH)5Yf&v~exp3o+sY4B^^b*eYkkCYl*T{*=-0HniSA_1F53eCb{x~1k3*`W zr~};p1A`k{1DV9=UPnLDgz{aJH=-LQo<5%+Em!DNN252xwIf*wF_zS^!(XSm(9eoj z=*dXG&n0>)_)N5oc6v!>-bd(2ragD8O=M|wGW z!xJQS<)u70m&6OmrF0WSsr@I%T*c#Qo#Ha4d3COcX+9}hM5!7JIGF>7<~C(Ear^Sn zm^ZFkV6~Ula6+8S?oOROOA6$C&q&dp`>oR-2Ym3(HT@O7Sd5c~+kjrmM)YmgPH*tL zX+znN>`tv;5eOfX?h{AuX^LK~V#gPCu=)Tigtq9&?7Xh$qN|%A$?V*v=&-2F$zTUv z`C#WyIrChS5|Kgm_GeudCFf;)!WH7FI60j^0o#65o6`w*S7R@)88n$1nrgU(oU0M9 zx+EuMkC>(4j1;m6NoGqEkpJYJ?vc|B zOlwT3t&UgL!pX_P*6g36`ZXQ; z9~Cv}ANFnJGp(;ZhS(@FT;3e)0)Kp;h^x;$*xZn*k0U6-&FwI=uOGaODdrsp-!K$Ac32^c{+FhI-HkYd5v=`PGsg%6I`4d9Jy)uW0y%) zm&j^9WBAp*P8#kGJUhB!L?a%h$hJgQrx!6KCB_TRo%9{t0J7KW8!o1B!NC)VGLM5! zpZy5Jc{`r{1e(jd%jsG7k%I+m#CGS*BPA65ZVW~fLYw0dA-H_}O zrkGFL&P1PG9p2(%QiEWm6x;U-U&I#;Em$nx-_I^wtgw3xUPVVu zqSuKnx&dIT-XT+T10p;yjo1Y)z(x1fb8Dzfn8e yu?e%!_ptzGB|8GrCfu%p?(_ zQccdaaVK$5bz;*rnyK{_SQYM>;aES6Qs^lj9lEs6_J+%nIiuQC*fN;z8md>r_~Mfl zU%p5Dt_YT>gQqfr@`cR!$NWr~+`CZb%dn;WtzrAOI>P_JtsB76PYe*<%H(y>qx-`Kq!X_; z<{RpAqYhE=L1r*M)gNF3B8r(<%8mo*SR2hu zccLRZwGARt)Hlo1euqTyM>^!HK*!Q2P;4UYrysje@;(<|$&%vQekbn|0Ruu_Io(w4#%p6ld2Yp7tlA`Y$cciThP zKzNGIMPXX%&Ud0uQh!uQZz|FB`4KGD?3!ND?wQt6!n*f4EmCoJUh&b?;B{|lxs#F- z31~HQ`SF4x$&v00@(P+j1pAaj5!s`)b2RDBp*PB=2IB>oBF!*6vwr7Dp%zpAx*dPr zb@Zjq^XjN?O4QcZ*O+8>)|HlrR>oD*?WQl5ri3R#2?*W6iJ>>kH%KnnME&TT@ZzrHS$Q%LC?n|e>V+D+8D zYc4)QddFz7I8#}y#Wj6>4P%34dZH~OUDb?uP%-E zwjXM(?Sg~1!|wI(RVuxbu)-rH+O=igSho_pDCw(c6b=P zKk4ATlB?bj9+HHlh<_!&z0rx13K3ZrAR8W)!@Y}o`?a*JJsD+twZIv`W)@Y?Amu_u zz``@-e2X}27$i(2=9rvIu5uTUOVhzwu%mNazS|lZb&PT;XE2|B&W1>=B58#*!~D&) zfVmJGg8UdP*fx(>Cj^?yS^zH#o-$Q-*$SnK(ZVFkw+er=>N^7!)FtP3y~Xxnu^nzY zikgB>Nj0%;WOltWIob|}%lo?_C7<``a5hEkx&1ku$|)i>Rh6@3h*`slY=9U}(Ql_< zaNG*J8vb&@zpdhAvv`?{=zDedJ23TD&Zg__snRAH4eh~^oawdYi6A3w8<Ozh@Kw)#bdktM^GVb zrG08?0bG?|NG+w^&JvD*7LAbjED{_Zkc`3H!My>0u5Q}m!+6VokMLXxl`Mkd=g&Xx z-a>m*#G3SLlhbKB!)tnzfWOBV;u;ftU}S!NdD5+YtOjLg?X}dl>7m^gOpihrf1;PY zvll&>dIuUGs{Qnd- zwIR3oIrct8Va^Tm0t#(bJD7c$Z7DO9*7NnRZorrSm`b`cxz>OIC;jSE3DO8`hX955ui`s%||YQtt2 z5DNA&pG-V+4oI2s*x^>-$6J?p=I>C|9wZF8z;VjR??Icg?1w2v5Me+FgAeGGa8(3S z4vg*$>zC-WIVZtJ7}o9{D-7d>zCe|z#<9>CFve-OPAYsneTb^JH!Enaza#j}^mXy1 z+ULn^10+rWLF6j2>Ya@@Kq?26>AqK{A_| zQKb*~F1>sE*=d?A?W7N2j?L09_7n+HGi{VY;MoTGr_)G9)ot$p!-UY5zZ2Xtbm=t z@dpPSGwgH=QtIcEulQNI>S-#ifbnO5EWkI;$A|pxJd885oM+ zGZ0_0gDvG8q2xebj+fbCHYfAXuZStH2j~|d^sBAzo46(K8n59+T6rzBwK)^rfPT+B zyIFw)9YC-V^rhtK`!3jrhmW-sTmM+tPH+;nwjL#-SjQPUZ53L@A>y*rt(#M(qsiB2 zx6B)dI}6Wlsw%bJ8h|(lhkJVogQZA&n{?Vgs6gNSXzuZpEyu*xySy8ro07QZ7Vk1!3tJphN_5V7qOiyK8p z#@jcDD8nmtYi1^l8ml;AF<#IPK?!pqf9D4moYk>d99Im}Jtwj6c#+A;f)CQ*f-hZ< z=p_T86jog%!p)D&5g9taSwYi&eP z#JuEK%+NULWus;0w32-SYFku#i}d~+{Pkho&^{;RxzP&0!RCm3-9K6`>KZpnzS6?L z^H^V*s!8<>x8bomvD%rh>Zp3>Db%kyin;qtl+jAv8Oo~1g~mqGAC&Qi_wy|xEt2iz zWAJEfTV%cl2Cs<1L&DLRVVH05EDq`pH7Oh7sR`NNkL%wi}8n>IXcO40hp+J+sC!W?!krJf!GJNE8uj zg-y~Ns-<~D?yqbzVRB}G>0A^f0!^N7l=$m0OdZuqAOQqLc zX?AEGr1Ht+inZ-Qiwnl@Z0qukd__a!C*CKuGdy5#nD7VUBM^6OCpxCa2A(X;e0&V4 zM&WR8+wErQ7UIc6LY~Q9x%Sn*Tn>>P`^t&idaOEnOd(Ufw#>NoR^1QdhJ8s`h^|R_ zXX`c5*O~Xdvh%q;7L!_!ohf$NfEBmCde|#uVZvEo>OfEq%+Ns7&_f$OR9xsihRpBb z+cjk8LyDm@U{YN>+r46?nn{7Gh(;WhFw6GAxtcKD+YWV?uge>;+q#Xx4!GpRkVZYu zzsF}1)7$?%s9g9CH=Zs+B%M_)+~*j3L0&Q9u7!|+T`^O{xE6qvAP?XWv9_MrZKdo& z%IyU)$Q95AB4!#hT!_dA>4e@zjOBD*Y=XjtMm)V|+IXzjuM;(l+8aA5#Kaz_$rR6! zj>#&^DidYD$nUY(D$mH`9eb|dtV0b{S>H6FBfq>t5`;OxA4Nn{J(+XihF(stSche7$es&~N$epi&PDM_N`As;*9D^L==2Q7Z2zD+CiU(|+-kL*VG+&9!Yb3LgPy?A zm7Z&^qRG_JIxK7-FBzZI3Q<;{`DIxtc48k> zc|0dmX;Z=W$+)qE)~`yn6MdoJ4co;%!`ddy+FV538Y)j(vg}5*k(WK)KWZ3WaOG!8 z!syGn=s{H$odtpqFrT#JGM*utN7B((abXnpDM6w56nhw}OY}0TiTG1#f*VFZr+^-g zbP10`$LPq_;PvrA1XXlyx2uM^mrjTzX}w{yuLo-cOClE8MMk47T25G8M!9Z5ypOSV zAJUBGEg5L2fY)ZGJb^E34R2zJ?}Vf>{~gB!8=5Z) z9y$>5c)=;o0HeHHSuE4U)#vG&KF|I%-cF6f$~pdYJWk_dD}iOA>iA$O$+4%@>JU08 zS`ep)$XLPJ+n0_i@PkF#ri6T8?ZeAot$6JIYHm&P6EB=BiaNY|aA$W0I+nz*zkz_z zkEru!tj!QUffq%)8y0y`T&`fuus-1p>=^hnBiBqD^hXrPs`PY9tU3m0np~rISY09> z`P3s=-kt_cYcxWd{de@}TwSqg*xVhp;E9zCsnXo6z z?f&Sv^U7n4`xr=mXle94HzOdN!2kB~4=%)u&N!+2;z6UYKUDqi-s6AZ!haB;@&B`? z_TRX0%@suz^TRdCb?!vNJYPY8L_}&07uySH9%W^Tc&1pia6y1q#?*Drf}GjGbPjBS zbOPcUY#*$3sL2x4v_i*Y=N7E$mR}J%|GUI(>WEr+28+V z%v5{#e!UF*6~G&%;l*q*$V?&r$Pp^sE^i-0$+RH3ERUUdQ0>rAq2(2QAbG}$y{de( z>{qD~GGuOk559Y@%$?N^1ApVL_a704>8OD%8Y%8B;FCt%AoPu8*D1 zLB5X>b}Syz81pn;xnB}%0FnwazlWfUV)Z-~rZg6~b z6!9J$EcE&sEbzcy?CI~=boWA&eeIa%z(7SE^qgVLz??1Vbc1*aRvc%Mri)AJaAG!p z$X!_9Ds;Zz)f+;%s&dRcJt2==P{^j3bf0M=nJd&xwUGlUFn?H=2W(*2I2Gdu zv!gYCwM10aeus)`RIZSrCK=&oKaO_Ry~D1B5!y0R=%!i2*KfXGYX&gNv_u+n9wiR5 z*e$Zjju&ODRW3phN925%S(jL+bCHv6rZtc?!*`1TyYXT6%Ju=|X;6D@lq$8T zW{Y|e39ioPez(pBH%k)HzFITXHvnD6hw^lIoUMA;qAJ^CU?top1fo@s7xT13Fvn1H z6JWa-6+FJF#x>~+A;D~;VDs26>^oH0EI`IYT2iagy23?nyJ==i{g4%HrAf1-*v zK1)~@&(KkwR7TL}L(A@C_S0G;-GMDy=MJn2$FP5s<%wC)4jC5PXoxrQBFZ_k0P{{s@sz+gX`-!=T8rcB(=7vW}^K6oLWMmp(rwDh}b zwaGGd>yEy6fHv%jM$yJXo5oMAQ>c9j`**}F?MCry;T@47@r?&sKHgVe$MCqk#Z_3S z1GZI~nOEN*P~+UaFGnj{{Jo@16`(qVNtbU>O0Hf57-P>x8Jikp=`s8xWs^dAJ9lCQ z)GFm+=OV%AMVqVATtN@|vp61VVAHRn87}%PC^RAzJ%JngmZTasWBAWsoAqBU+8L8u z4A&Pe?fmTm0?mK-BL9t+{y7o(7jm+RpOhL9KnY#E&qu^}B6=K_dB}*VlSEiC9fn)+V=J;OnN)Ta5v66ic1rG+dGAJ1 z1%Zb_+!$=tQ~lxQrzv3x#CPb?CekEkA}0MYSgx$Jdd}q8+R=ma$|&1a#)TQ=l$1tQ z=tL9&_^vJ)Pk}EDO-va`UCT1m#Uty1{v^A3P~83_#v^ozH}6*9mIjIr;t3Uv%@VeW zGL6(CwCUp)Jq%G0bIG%?{_*Y#5IHf*5M@wPo6A{$Um++Co$wLC=J1aoG93&T7Ho}P z=mGEPP7GbvoG!uD$k(H3A$Z))+i{Hy?QHdk>3xSBXR0j!11O^mEe9RHmw!pvzv?Ua~2_l2Yh~_!s1qS`|0~0)YsbHSz8!mG)WiJE| z2f($6TQtt6L_f~ApQYQKSb=`053LgrQq7G@98#igV>y#i==-nEjQ!XNu9 z~;mE+gtj4IDDNQJ~JVk5Ux6&LCSFL!y=>79kE9=V}J7tD==Ga+IW zX)r7>VZ9dY=V&}DR))xUoV!u(Z|%3ciQi_2jl}3=$Agc(`RPb z8kEBpvY>1FGQ9W$n>Cq=DIpski};nE)`p3IUw1Oz0|wxll^)4dq3;CCY@RyJgFgc# zKouFh!`?Xuo{IMz^xi-h=StCis_M7yq$u) z?XHvw*HP0VgR+KR6wI)jEMX|ssqYvSf*_3W8zVTQzD?3>H!#>InzpSO)@SC8q*ii- z%%h}_#0{4JG;Jm`4zg};BPTGkYamx$Xo#O~lBirRY)q=5M45n{GCfV7h9qwyu1NxOMoP4)jjZMxmT|IQQh0U7C$EbnMN<3)Kk?fFHYq$d|ICu>KbY_hO zTZM+uKHe(cIZfEqyzyYSUBZa8;Fcut-GN!HSA9ius`ltNebF46ZX_BbZNU}}ZOm{M2&nANL9@0qvih15(|`S~z}m&h!u4x~(%MAO$jHRWNfuxWF#B)E&g3ghSQ9|> z(MFaLQj)NE0lowyjvg8z0#m6FIuKE9lDO~Glg}nSb7`~^&#(Lw{}GVOS>U)m8bF}x zVjbXljBm34Cs-yM6TVusr+3kYFjr28STT3g056y3cH5Tmge~ASxBj z%|yb>$eF;WgrcOZf569sDZOVwoo%8>XO>XQOX1OyN9I-SQgrm;U;+#3OI(zrWyow3 zk==|{lt2xrQ%FIXOTejR>;wv(Pb8u8}BUpx?yd(Abh6? zsoO3VYWkeLnF43&@*#MQ9-i-d0t*xN-UEyNKeyNMHw|A(k(_6QKO=nKMCxD(W(Yop zsRQ)QeL4X3Lxp^L%wzi2-WVSsf61dqliPUM7srDB?Wm6Lzn0&{*}|IsKQW;02(Y&| zaTKv|`U(pSzuvR6Rduu$wzK_W-Y-7>7s?G$)U}&uK;<>vU}^^ns@Z!p+9?St1s)dG zK%y6xkPyyS1$~&6v{kl?Md6gwM|>mt6Upm>oa8RLD^8T{0?HC!Z>;(Bob7el(DV6x zi`I)$&E&ngwFS@bi4^xFLAn`=fzTC;aimE^!cMI2n@Vo%Ae-ne`RF((&5y6xsjjAZ zVguVoQ?Z9uk$2ON;ersE%PU*xGO@T*;j1BO5#TuZKEf(mB7|g7pcEA=nYJ{s3vlbg zd4-DUlD{*6o%Gc^N!Nptgay>j6E5;3psI+C3Q!1ZIbeCubW%w4pq9)MSDyB{HLm|k zxv-{$$A*pS@csolri$Ge<4VZ}e~78JOL-EVyrbxKra^d{?|NnPp86!q>t<&IP07?Z z^>~IK^k#OEKgRH+LjllZXk7iA>2cfH6+(e&9ku5poo~6y{GC5>(bRK7hwjiurqAiZ zg*DmtgY}v83IjE&AbiWgMyFbaRUPZ{lYiz$U^&Zt2YjG<%m((&_JUbZcfJ22(>bi5 z!J?<7AySj0JZ&<-qXX;mcV!f~>G=sB0KnjWca4}vrtunD^1TrpfeS^4dvFr!65knK zZh`d;*VOkPs4*-9kL>$GP0`(M!j~B;#x?Ba~&s6CopvO86oM?-? zOw#dIRc;6A6T?B`Qp%^<U5 z19x(ywSH$_N+Io!6;e?`tWaM$`=Db!gzx|lQ${DG!zb1Zl&|{kX0y6xvO1o z220r<-oaS^^R2pEyY;=Qllqpmue|5yI~D|iI!IGt@iod{Opz@*ml^w2bNs)p`M(Io z|E;;m*Xpjd9l)4G#KaWfV(t8YUn@A;nK^#xgv=LtnArX|vWQVuw3}B${h+frU2>9^ z!l6)!Uo4`5k`<<;E(ido7M6lKTgWezNLq>U*=uz&s=cc$1%>VrAeOoUtA|T6gO4>UNqsdK=NF*8|~*sl&wI=x9-EGiq*aqV!(VVXA57 zw9*o6Ir8Lj1npUXvlevtn(_+^X5rzdR>#(}4YcB9O50q97%rW2me5_L=%ffYPUSRc z!vv?Kv>dH994Qi>U(a<0KF6NH5b16enCp+mw^Hb3Xs1^tThFpz!3QuN#}KBbww`(h z7GO)1olDqy6?T$()R7y%NYx*B0k_2IBiZ14&8|JPFxeMF{vSTxF-Vi3+ZOI=Thq2} zyQgjYY1_7^ZQHh{?P))4+qUiQJLi1&{yE>h?~jU%tjdV0h|FENbM3X(KnJdPKc?~k zh=^Ixv*+smUll!DTWH!jrV*wSh*(mx0o6}1@JExzF(#9FXgmTXVoU+>kDe68N)dkQ zH#_98Zv$}lQwjKL@yBd;U(UD0UCl322=pav<=6g>03{O_3oKTq;9bLFX1ia*lw;#K zOiYDcBJf)82->83N_Y(J7Kr_3lE)hAu;)Q(nUVydv+l+nQ$?|%MWTy`t>{havFSQloHwiIkGK9YZ79^9?AZo0ZyQlVR#}lF%dn5n%xYksXf8gnBm=wO7g_^! zauQ-bH1Dc@3ItZ-9D_*pH}p!IG7j8A_o94#~>$LR|TFq zZ-b00*nuw|-5C2lJDCw&8p5N~Z1J&TrcyErds&!l3$eSz%`(*izc;-?HAFD9AHb-| z>)id`QCrzRws^9(#&=pIx9OEf2rmlob8sK&xPCWS+nD~qzU|qG6KwA{zbikcfQrdH z+ zQg>O<`K4L8rN7`GJB0*3<3`z({lWe#K!4AZLsI{%z#ja^OpfjU{!{)x0ZH~RB0W5X zTwN^w=|nA!4PEU2=LR05x~}|B&ZP?#pNgDMwD*ajI6oJqv!L81gu=KpqH22avXf0w zX3HjbCI!n9>l046)5rr5&v5ja!xkKK42zmqHzPx$9Nn_MZk`gLeSLgC=LFf;H1O#B zn=8|^1iRrujHfbgA+8i<9jaXc;CQBAmQvMGQPhFec2H1knCK2x!T`e6soyrqCamX% zTQ4dX_E*8so)E*TB$*io{$c6X)~{aWfaqdTh=xEeGvOAN9H&-t5tEE-qso<+C!2>+ zskX51H-H}#X{A75wqFe-J{?o8Bx|>fTBtl&tcbdR|132Ztqu5X0i-pisB-z8n71%q%>EF}yy5?z=Ve`}hVh{Drv1YWL zW=%ug_&chF11gDv3D6B)Tz5g54H0mDHNjuKZ+)CKFk4Z|$RD zfRuKLW`1B>B?*RUfVd0+u8h3r-{@fZ{k)c!93t1b0+Q9vOaRnEn1*IL>5Z4E4dZ!7 ztp4GP-^1d>8~LMeb}bW!(aAnB1tM_*la=Xx)q(I0Y@__Zd$!KYb8T2VBRw%e$iSdZ zkwdMwd}eV9q*;YvrBFTv1>1+}{H!JK2M*C|TNe$ZSA>UHKk);wz$(F$rXVc|sI^lD zV^?_J!3cLM;GJuBMbftbaRUs$;F}HDEDtIeHQ)^EJJ1F9FKJTGH<(Jj`phE6OuvE) zqK^K`;3S{Y#1M@8yRQwH`?kHMq4tHX#rJ>5lY3DM#o@or4&^_xtBC(|JpGTfrbGkA z2Tu+AyT^pHannww!4^!$5?@5v`LYy~T`qs7SYt$JgrY(w%C+IWA;ZkwEF)u5sDvOK zGk;G>Mh&elvXDcV69J_h02l&O;!{$({fng9Rlc3ID#tmB^FIG^w{HLUpF+iB`|
NnX)EH+Nua)3Y(c z&{(nX_ht=QbJ%DzAya}!&uNu!4V0xI)QE$SY__m)SAKcN0P(&JcoK*Lxr@P zY&P=}&B3*UWNlc|&$Oh{BEqwK2+N2U$4WB7Fd|aIal`FGANUa9E-O)!gV`((ZGCc$ zBJA|FFrlg~9OBp#f7aHodCe{6= zay$6vN~zj1ddMZ9gQ4p32(7wD?(dE>KA2;SOzXRmPBiBc6g`eOsy+pVcHu=;Yd8@{ zSGgXf@%sKKQz~;!J;|2fC@emm#^_rnO0esEn^QxXgJYd`#FPWOUU5b;9eMAF zZhfiZb|gk8aJIw*YLp4!*(=3l8Cp{(%p?ho22*vN9+5NLV0TTazNY$B5L6UKUrd$n zjbX%#m7&F#U?QNOBXkiiWB*_tk+H?N3`vg;1F-I+83{M2!8<^nydGr5XX}tC!10&e z7D36bLaB56WrjL&HiiMVtpff|K%|*{t*ltt^5ood{FOG0<>k&1h95qPio)2`eL${YAGIx(b4VN*~nKn6E~SIQUuRH zQ+5zP6jfnP$S0iJ@~t!Ai3o`X7biohli;E zT#yXyl{bojG@-TGZzpdVDXhbmF%F9+-^YSIv|MT1l3j zrxOFq>gd2%U}?6}8mIj?M zc077Zc9fq(-)4+gXv?Az26IO6eV`RAJz8e3)SC7~>%rlzDwySVx*q$ygTR5kW2ds- z!HBgcq0KON9*8Ff$X0wOq$`T7ml(@TF)VeoF}x1OttjuVHn3~sHrMB++}f7f9H%@f z=|kP_?#+fve@{0MlbkC9tyvQ_R?lRdRJ@$qcB(8*jyMyeME5ns6ypVI1Xm*Zr{DuS zZ!1)rQfa89c~;l~VkCiHI|PCBd`S*2RLNQM8!g9L6?n`^evQNEwfO@&JJRme+uopQX0%Jo zgd5G&#&{nX{o?TQwQvF1<^Cg3?2co;_06=~Hcb6~4XWpNFL!WU{+CK;>gH%|BLOh7@!hsa(>pNDAmpcuVO-?;Bic17R}^|6@8DahH)G z!EmhsfunLL|3b=M0MeK2vqZ|OqUqS8npxwge$w-4pFVXFq$_EKrZY?BuP@Az@(k`L z`ViQBSk`y+YwRT;&W| z2e3UfkCo^uTA4}Qmmtqs+nk#gNr2W4 zTH%hhErhB)pkXR{B!q5P3-OM+M;qu~f>}IjtF%>w{~K-0*jPVLl?Chz&zIdxp}bjx zStp&Iufr58FTQ36AHU)0+CmvaOpKF;W@sMTFpJ`j;3d)J_$tNQI^c<^1o<49Z(~K> z;EZTBaVT%14(bFw2ob@?JLQ2@(1pCdg3S%E4*dJ}dA*v}_a4_P(a`cHnBFJxNobAv zf&Zl-Yt*lhn-wjZsq<9v-IsXxAxMZ58C@e0!rzhJ+D@9^3~?~yllY^s$?&oNwyH!#~6x4gUrfxplCvK#!f z$viuszW>MFEcFL?>ux*((!L$;R?xc*myjRIjgnQX79@UPD$6Dz0jutM@7h_pq z0Zr)#O<^y_K6jfY^X%A-ip>P%3saX{!v;fxT-*0C_j4=UMH+Xth(XVkVGiiKE#f)q z%Jp=JT)uy{&}Iq2E*xr4YsJ5>w^=#-mRZ4vPXpI6q~1aFwi+lQcimO45V-JXP;>(Q zo={U`{=_JF`EQj87Wf}{Qy35s8r1*9Mxg({CvOt}?Vh9d&(}iI-quvs-rm~P;eRA@ zG5?1HO}puruc@S{YNAF3vmUc2B4!k*yi))<5BQmvd3tr}cIs#9)*AX>t`=~{f#Uz0 z0&Nk!7sSZwJe}=)-R^$0{yeS!V`Dh7w{w5rZ9ir!Z7Cd7dwZcK;BT#V0bzTt>;@Cl z#|#A!-IL6CZ@eHH!CG>OO8!%G8&8t4)Ro@}USB*k>oEUo0LsljsJ-%5Mo^MJF2I8- z#v7a5VdJ-Cd%(a+y6QwTmi+?f8Nxtm{g-+WGL>t;s#epv7ug>inqimZCVm!uT5Pf6 ziEgQt7^%xJf#!aPWbuC_3Nxfb&CFbQy!(8ANpkWLI4oSnH?Q3f?0k1t$3d+lkQs{~(>06l&v|MpcFsyAv zin6N!-;pggosR*vV=DO(#+}4ps|5$`udE%Kdmp?G7B#y%H`R|i8skKOd9Xzx8xgR$>Zo2R2Ytktq^w#ul4uicxW#{ zFjG_RNlBroV_n;a7U(KIpcp*{M~e~@>Q#Av90Jc5v%0c>egEdY4v3%|K1XvB{O_8G zkTWLC>OZKf;XguMH2-Pw{BKbFzaY;4v2seZV0>^7Q~d4O=AwaPhP3h|!hw5aqOtT@ z!SNz}$of**Bl3TK209@F=Tn1+mgZa8yh(Png%Zd6Mt}^NSjy)etQrF zme*llAW=N_8R*O~d2!apJnF%(JcN??=`$qs3Y+~xs>L9x`0^NIn!8mMRFA_tg`etw z3k{9JAjnl@ygIiJcNHTy02GMAvBVqEss&t2<2mnw!; zU`J)0>lWiqVqo|ex7!+@0i>B~BSU1A_0w#Ee+2pJx0BFiZ7RDHEvE*ptc9md(B{&+ zKE>TM)+Pd>HEmdJao7U@S>nL(qq*A)#eLOuIfAS@j`_sK0UEY6OAJJ-kOrHG zjHx`g!9j*_jRcJ%>CE9K2MVf?BUZKFHY?EpV6ai7sET-tqk=nDFh-(65rhjtlKEY% z@G&cQ<5BKatfdA1FKuB=i>CCC5(|9TMW%K~GbA4}80I5%B}(gck#Wlq@$nO3%@QP_ z8nvPkJFa|znk>V92cA!K1rKtr)skHEJD;k8P|R8RkCq1Rh^&}Evwa4BUJz2f!2=MH zo4j8Y$YL2313}H~F7@J7mh>u%556Hw0VUOz-Un@ZASCL)y8}4XXS`t1AC*^>PLwIc zUQok5PFS=*#)Z!3JZN&eZ6ZDP^-c@StY*t20JhCnbMxXf=LK#;`4KHEqMZ-Ly9KsS zI2VUJGY&PmdbM+iT)zek)#Qc#_i4uH43 z@T5SZBrhNCiK~~esjsO9!qBpaWK<`>!-`b71Y5ReXQ4AJU~T2Njri1CEp5oKw;Lnm)-Y@Z3sEY}XIgSy%xo=uek(kAAH5MsV$V3uTUsoTzxp_rF=tx zV07vlJNKtJhCu`b}*#m&5LV4TAE&%KtHViDAdv#c^x`J7bg z&N;#I2GkF@SIGht6p-V}`!F_~lCXjl1BdTLIjD2hH$J^YFN`7f{Q?OHPFEM$65^!u zNwkelo*5+$ZT|oQ%o%;rBX$+?xhvjb)SHgNHE_yP%wYkkvXHS{Bf$OiKJ5d1gI0j< zF6N}Aq=(WDo(J{e-uOecxPD>XZ@|u-tgTR<972`q8;&ZD!cep^@B5CaqFz|oU!iFj zU0;6fQX&~15E53EW&w1s9gQQ~Zk16X%6 zjG`j0yq}4deX2?Tr(03kg>C(!7a|b9qFI?jcE^Y>-VhudI@&LI6Qa}WQ>4H_!UVyF z((cm&!3gmq@;BD#5P~0;_2qgZhtJS|>WdtjY=q zLnHH~Fm!cxw|Z?Vw8*~?I$g#9j&uvgm7vPr#&iZgPP~v~BI4jOv;*OQ?jYJtzO<^y z7-#C={r7CO810!^s(MT!@@Vz_SVU)7VBi(e1%1rvS!?PTa}Uv`J!EP3s6Y!xUgM^8 z4f!fq<3Wer_#;u!5ECZ|^c1{|q_lh3m^9|nsMR1#Qm|?4Yp5~|er2?W^7~cl;_r4WSme_o68J9p03~Hc%X#VcX!xAu%1`R!dfGJCp zV*&m47>s^%Ib0~-2f$6oSgn3jg8m%UA;ArcdcRyM5;}|r;)?a^D*lel5C`V5G=c~k zy*w_&BfySOxE!(~PI$*dwG><+-%KT5p?whOUMA*k<9*gi#T{h3DAxzAPxN&Xws8o9Cp*`PA5>d9*Z-ynV# z9yY*1WR^D8|C%I@vo+d8r^pjJ$>eo|j>XiLWvTWLl(^;JHCsoPgem6PvegHb-OTf| zvTgsHSa;BkbG=(NgPO|CZu9gUCGr$8*EoH2_Z#^BnxF0yM~t`|9ws_xZ8X8iZYqh! zAh;HXJ)3P&)Q0(&F>!LN0g#bdbis-cQxyGn9Qgh`q+~49Fqd2epikEUw9caM%V6WgP)532RMRW}8gNS%V%Hx7apSz}tn@bQy!<=lbhmAH=FsMD?leawbnP5BWM0 z5{)@EEIYMu5;u)!+HQWhQ;D3_Cm_NADNeb-f56}<{41aYq8p4=93d=-=q0Yx#knGYfXVt z+kMxlus}t2T5FEyCN~!}90O_X@@PQpuy;kuGz@bWft%diBTx?d)_xWd_-(!LmVrh**oKg!1CNF&LX4{*j|) zIvjCR0I2UUuuEXh<9}oT_zT#jOrJAHNLFT~Ilh9hGJPI1<5`C-WA{tUYlyMeoy!+U zhA#=p!u1R7DNg9u4|QfED-2TuKI}>p#2P9--z;Bbf4Op*;Q9LCbO&aL2i<0O$ByoI z!9;Ght733FC>Pz>$_mw(F`zU?`m@>gE`9_p*=7o=7av`-&ifU(^)UU`Kg3Kw`h9-1 z6`e6+im=|m2v`pN(2dE%%n8YyQz;#3Q-|x`91z?gj68cMrHl}C25|6(_dIGk*8cA3 zRHB|Nwv{@sP4W+YZM)VKI>RlB`n=Oj~Rzx~M+Khz$N$45rLn6k1nvvD^&HtsMA4`s=MmuOJID@$s8Ph4E zAmSV^+s-z8cfv~Yd(40Sh4JG#F~aB>WFoX7ykaOr3JaJ&Lb49=B8Vk-SQT9%7TYhv z?-Pprt{|=Y5ZQ1?od|A<_IJU93|l4oAfBm?3-wk{O<8ea+`}u%(kub(LFo2zFtd?4 zwpN|2mBNywv+d^y_8#<$r>*5+$wRTCygFLcrwT(qc^n&@9r+}Kd_u@Ithz(6Qb4}A zWo_HdBj#V$VE#l6pD0a=NfB0l^6W^g`vm^sta>Tly?$E&{F?TTX~DsKF~poFfmN%2 z4x`Dc{u{Lkqz&y!33;X}weD}&;7p>xiI&ZUb1H9iD25a(gI|`|;G^NwJPv=1S5e)j z;U;`?n}jnY6rA{V^ zxTd{bK)Gi^odL3l989DQlN+Zs39Xe&otGeY(b5>rlIqfc7Ap4}EC?j<{M=hlH{1+d zw|c}}yx88_xQr`{98Z!d^FNH77=u(p-L{W6RvIn40f-BldeF-YD>p6#)(Qzf)lfZj z?3wAMtPPp>vMehkT`3gToPd%|D8~4`5WK{`#+}{L{jRUMt zrFz+O$C7y8$M&E4@+p+oV5c%uYzbqd2Y%SSgYy#xh4G3hQv>V*BnuKQhBa#=oZB~w{azUB+q%bRe_R^ z>fHBilnRTUfaJ201czL8^~Ix#+qOHSO)A|xWLqOxB$dT2W~)e-r9;bm=;p;RjYahB z*1hegN(VKK+ztr~h1}YP@6cfj{e#|sS`;3tJhIJK=tVJ-*h-5y9n*&cYCSdg#EHE# zSIx=r#qOaLJoVVf6v;(okg6?*L_55atl^W(gm^yjR?$GplNP>BZsBYEf_>wM0Lc;T zhf&gpzOWNxS>m+mN92N0{;4uw`P+9^*|-1~$uXpggj4- z^SFc4`uzj2OwdEVT@}Q`(^EcQ_5(ZtXTql*yGzdS&vrS_w>~~ra|Nb5abwf}Y!uq6R5f&6g2ge~2p(%c< z@O)cz%%rr4*cRJ5f`n@lvHNk@lE1a*96Kw6lJ~B-XfJW%?&-y?;E&?1AacU@`N`!O z6}V>8^%RZ7SQnZ-z$(jsX`amu*5Fj8g!3RTRwK^`2_QHe;_2y_n|6gSaGyPmI#kA0sYV<_qOZc#-2BO%hX)f$s-Z3xlI!ub z^;3ru11DA`4heAu%}HIXo&ctujzE2!6DIGE{?Zs>2}J+p&C$rc7gJC35gxhflorvsb%sGOxpuWhF)dL_&7&Z99=5M0b~Qa;Mo!j&Ti_kXW!86N%n= zSC@6Lw>UQ__F&+&Rzv?gscwAz8IP!n63>SP)^62(HK98nGjLY2*e^OwOq`3O|C92? z;TVhZ2SK%9AGW4ZavTB9?)mUbOoF`V7S=XM;#3EUpR+^oHtdV!GK^nXzCu>tpR|89 zdD{fnvCaN^^LL%amZ^}-E+214g&^56rpdc@yv0b<3}Ys?)f|fXN4oHf$six)-@<;W&&_kj z-B}M5U*1sb4)77aR=@%I?|Wkn-QJVuA96an25;~!gq(g1@O-5VGo7y&E_srxL6ZfS z*R%$gR}dyONgju*D&?geiSj7SZ@ftyA|}(*Y4KbvU!YLsi1EDQQCnb+-cM=K1io78o!v*);o<XwjaQH%)uIP&Zm?)Nfbfn;jIr z)d#!$gOe3QHp}2NBak@yYv3m(CPKkwI|{;d=gi552u?xj9ObCU^DJFQp4t4e1tPzM zvsRIGZ6VF+{6PvqsplMZWhz10YwS={?`~O0Ec$`-!klNUYtzWA^f9m7tkEzCy<_nS z=&<(awFeZvt51>@o_~>PLs05CY)$;}Oo$VDO)?l-{CS1Co=nxjqben*O1BR>#9`0^ zkwk^k-wcLCLGh|XLjdWv0_Hg54B&OzCE^3NCP}~OajK-LuRW53CkV~Su0U>zN%yQP zH8UH#W5P3-!ToO-2k&)}nFe`t+mdqCxxAHgcifup^gKpMObbox9LFK;LP3}0dP-UW z?Zo*^nrQ6*$FtZ(>kLCc2LY*|{!dUn$^RW~m9leoF|@Jy|M5p-G~j%+P0_#orRKf8 zvuu5<*XO!B?1E}-*SY~MOa$6c%2cM+xa8}_8x*aVn~57v&W(0mqN1W`5a7*VN{SUH zXz98DDyCnX2EPl-`Lesf`=AQT%YSDb`$%;(jUTrNen$NPJrlpPDP}prI>Ml!r6bCT;mjsg@X^#&<}CGf0JtR{Ecwd&)2zuhr#nqdgHj+g2n}GK9CHuwO zk>oZxy{vcOL)$8-}L^iVfJHAGfwN$prHjYV0ju}8%jWquw>}_W6j~m<}Jf!G?~r5&Rx)!9JNX!ts#SGe2HzobV5); zpj@&`cNcO&q+%*<%D7za|?m5qlmFK$=MJ_iv{aRs+BGVrs)98BlN^nMr{V_fcl_;jkzRju+c-y?gqBC_@J0dFLq-D9@VN&-`R9U;nv$Hg?>$oe4N&Ht$V_(JR3TG^! zzJsbQbi zFE6-{#9{G{+Z}ww!ycl*7rRdmU#_&|DqPfX3CR1I{Kk;bHwF6jh0opI`UV2W{*|nn zf_Y@%wW6APb&9RrbEN=PQRBEpM(N1w`81s=(xQj6 z-eO0k9=Al|>Ej|Mw&G`%q8e$2xVz1v4DXAi8G};R$y)ww638Y=9y$ZYFDM$}vzusg zUf+~BPX>(SjA|tgaFZr_e0{)+z9i6G#lgt=F_n$d=beAt0Sa0a7>z-?vcjl3e+W}+ z1&9=|vC=$co}-Zh*%3588G?v&U7%N1Qf-wNWJ)(v`iO5KHSkC5&g7CrKu8V}uQGcfcz zmBz#Lbqwqy#Z~UzHgOQ;Q-rPxrRNvl(&u6ts4~0=KkeS;zqURz%!-ERppmd%0v>iRlEf+H$yl{_8TMJzo0 z>n)`On|7=WQdsqhXI?#V{>+~}qt-cQbokEbgwV3QvSP7&hK4R{Z{aGHVS3;+h{|Hz z6$Js}_AJr383c_+6sNR|$qu6dqHXQTc6?(XWPCVZv=)D#6_;D_8P-=zOGEN5&?~8S zl5jQ?NL$c%O)*bOohdNwGIKM#jSAC?BVY={@A#c9GmX0=T(0G}xs`-%f3r=m6-cpK z!%waekyAvm9C3%>sixdZj+I(wQlbB4wv9xKI*T13DYG^T%}zZYJ|0$Oj^YtY+d$V$ zAVudSc-)FMl|54n=N{BnZTM|!>=bhaja?o7s+v1*U$!v!qQ%`T-6fBvmdPbVmro&d zk07TOp*KuxRUSTLRrBj{mjsnF8`d}rMViY8j`jo~Hp$fkv9F_g(jUo#Arp;Xw0M$~ zRIN!B22~$kx;QYmOkos@%|5k)!QypDMVe}1M9tZfkpXKGOxvKXB!=lo`p?|R1l=tA zp(1}c6T3Fwj_CPJwVsYtgeRKg?9?}%oRq0F+r+kdB=bFUdVDRPa;E~~>2$w}>O>v=?|e>#(-Lyx?nbg=ckJ#5U6;RT zNvHhXk$P}m9wSvFyU3}=7!y?Y z=fg$PbV8d7g25&-jOcs{%}wTDKm>!Vk);&rr;O1nvO0VrU&Q?TtYVU=ir`te8SLlS zKSNmV=+vF|ATGg`4$N1uS|n??f}C_4Sz!f|4Ly8#yTW-FBfvS48Tef|-46C(wEO_%pPhUC5$-~Y?!0vFZ^Gu`x=m7X99_?C-`|h zfmMM&Y@zdfitA@KPw4Mc(YHcY1)3*1xvW9V-r4n-9ZuBpFcf{yz+SR{ zo$ZSU_|fgwF~aakGr(9Be`~A|3)B=9`$M-TWKipq-NqRDRQc}ABo*s_5kV%doIX7LRLRau_gd@Rd_aLFXGSU+U?uAqh z8qusWWcvgQ&wu{|sRXmv?sl=xc<$6AR$+cl& zFNh5q1~kffG{3lDUdvEZu5c(aAG~+64FxdlfwY^*;JSS|m~CJusvi-!$XR`6@XtY2 znDHSz7}_Bx7zGq-^5{stTRy|I@N=>*y$zz>m^}^{d&~h;0kYiq8<^Wq7Dz0w31ShO^~LUfW6rfitR0(=3;Uue`Y%y@ex#eKPOW zO~V?)M#AeHB2kovn1v=n^D?2{2jhIQd9t|_Q+c|ZFaWt+r&#yrOu-!4pXAJuxM+Cx z*H&>eZ0v8Y`t}8{TV6smOj=__gFC=eah)mZt9gwz>>W$!>b3O;Rm^Ig*POZP8Rl0f zT~o=Nu1J|lO>}xX&#P58%Yl z83`HRs5#32Qm9mdCrMlV|NKNC+Z~ z9OB8xk5HJ>gBLi+m@(pvpw)1(OaVJKs*$Ou#@Knd#bk+V@y;YXT?)4eP9E5{J%KGtYinNYJUH9PU3A}66c>Xn zZ{Bn0<;8$WCOAL$^NqTjwM?5d=RHgw3!72WRo0c;+houoUA@HWLZM;^U$&sycWrFd zE7ekt9;kb0`lps{>R(}YnXlyGY}5pPd9zBpgXeJTY_jwaJGSJQC#-KJqmh-;ad&F- z-Y)E>!&`Rz!HtCz>%yOJ|v(u7P*I$jqEY3}(Z-orn4 zlI?CYKNl`6I){#2P1h)y(6?i;^z`N3bxTV%wNvQW+eu|x=kbj~s8rhCR*0H=iGkSj zk23lr9kr|p7#qKL=UjgO`@UnvzU)`&fI>1Qs7ubq{@+lK{hH* zvl6eSb9%yngRn^T<;jG1SVa)eA>T^XX=yUS@NCKpk?ovCW1D@!=@kn;l_BrG;hOTC z6K&H{<8K#dI(A+zw-MWxS+~{g$tI7|SfP$EYKxA}LlVO^sT#Oby^grkdZ^^lA}uEF zBSj$weBJG{+Bh@Yffzsw=HyChS(dtLE3i*}Zj@~!_T-Ay7z=B)+*~3|?w`Zd)Co2t zC&4DyB!o&YgSw+fJn6`sn$e)29`kUwAc+1MND7YjV%lO;H2}fNy>hD#=gT ze+-aFNpyKIoXY~Vq-}OWPBe?Rfu^{ps8>Xy%42r@RV#*QV~P83jdlFNgkPN=T|Kt7 zV*M`Rh*30&AWlb$;ae130e@}Tqi3zx2^JQHpM>j$6x`#{mu%tZlwx9Gj@Hc92IuY* zarmT|*d0E~vt6<+r?W^UW0&#U&)8B6+1+;k^2|FWBRP9?C4Rk)HAh&=AS8FS|NQaZ z2j!iZ)nbEyg4ZTp-zHwVlfLC~tXIrv(xrP8PAtR{*c;T24ycA-;auWsya-!kF~CWZ zw_uZ|%urXgUbc@x=L=_g@QJ@m#5beS@6W195Hn7>_}z@Xt{DIEA`A&V82bc^#!q8$ zFh?z_Vn|ozJ;NPd^5uu(9tspo8t%&-U9Ckay-s@DnM*R5rtu|4)~e)`z0P-sy?)kc zs_k&J@0&0!q4~%cKL)2l;N*T&0;mqX5T{Qy60%JtKTQZ-xb%KOcgqwJmb%MOOKk7N zgq})R_6**{8A|6H?fO+2`#QU)p$Ei2&nbj6TpLSIT^D$|`TcSeh+)}VMb}LmvZ{O| ze*1IdCt3+yhdYVxcM)Q_V0bIXLgr6~%JS<<&dxIgfL=Vnx4YHuU@I34JXA|+$_S3~ zy~X#gO_X!cSs^XM{yzDGNM>?v(+sF#<0;AH^YrE8smx<36bUsHbN#y57K8WEu(`qHvQ6cAZPo=J5C(lSmUCZ57Rj6cx!e^rfaI5%w}unz}4 zoX=nt)FVNV%QDJH`o!u9olLD4O5fl)xp+#RloZlaA92o3x4->?rB4`gS$;WO{R;Z3>cG3IgFX2EA?PK^M}@%1%A;?f6}s&CV$cIyEr#q5;yHdNZ9h{| z-=dX+a5elJoDo?Eq&Og!nN6A)5yYpnGEp}?=!C-V)(*~z-+?kY1Q7qs#Rsy%hu_60rdbB+QQNr?S1 z?;xtjUv|*E3}HmuNyB9aFL5H~3Ho0UsmuMZELp1a#CA1g`P{-mT?BchuLEtK}!QZ=3AWakRu~?f9V~3F;TV`5%9Pcs_$gq&CcU}r8gOO zC2&SWPsSG{&o-LIGTBqp6SLQZPvYKp$$7L4WRRZ0BR$Kf0I0SCFkqveCp@f)o8W)! z$%7D1R`&j7W9Q9CGus_)b%+B#J2G;l*FLz#s$hw{BHS~WNLODV#(!u_2Pe&tMsq={ zdm7>_WecWF#D=?eMjLj=-_z`aHMZ=3_-&E8;ibPmM}61i6J3is*=dKf%HC>=xbj4$ zS|Q-hWQ8T5mWde6h@;mS+?k=89?1FU<%qH9B(l&O>k|u_aD|DY*@~(`_pb|B#rJ&g zR0(~(68fpUPz6TdS@4JT5MOPrqDh5_H(eX1$P2SQrkvN8sTxwV>l0)Qq z0pzTuvtEAKRDkKGhhv^jk%|HQ1DdF%5oKq5BS>szk-CIke{%js?~%@$uaN3^Uz6Wf z_iyx{bZ(;9y4X&>LPV=L=d+A}7I4GkK0c1Xts{rrW1Q7apHf-))`BgC^0^F(>At1* za@e7{lq%yAkn*NH8Q1{@{lKhRg*^TfGvv!Sn*ed*x@6>M%aaqySxR|oNadYt1mpUZ z6H(rupHYf&Z z29$5g#|0MX#aR6TZ$@eGxxABRKakDYtD%5BmKp;HbG_ZbT+=81E&=XRk6m_3t9PvD zr5Cqy(v?gHcYvYvXkNH@S#Po~q(_7MOuCAB8G$a9BC##gw^5mW16cML=T=ERL7wsk zzNEayTG?mtB=x*wc@ifBCJ|irFVMOvH)AFRW8WE~U()QT=HBCe@s$dA9O!@`zAAT) zaOZ7l6vyR+Nk_OOF!ZlZmjoImKh)dxFbbR~z(cMhfeX1l7S_`;h|v3gI}n9$sSQ>+3@AFAy9=B_y$)q;Wdl|C-X|VV3w8 z2S#>|5dGA8^9%Bu&fhmVRrTX>Z7{~3V&0UpJNEl0=N32euvDGCJ>#6dUSi&PxFW*s zS`}TB>?}H(T2lxBJ!V#2taV;q%zd6fOr=SGHpoSG*4PDaiG0pdb5`jelVipkEk%FV zThLc@Hc_AL1#D&T4D=w@UezYNJ%0=f3iVRuVL5H?eeZM}4W*bomebEU@e2d`M<~uW zf#Bugwf`VezG|^Qbt6R_=U0}|=k;mIIakz99*>FrsQR{0aQRP6ko?5<7bkDN8evZ& zB@_KqQG?ErKL=1*ZM9_5?Pq%lcS4uLSzN(Mr5=t6xHLS~Ym`UgM@D&VNu8e?_=nSFtF$u@hpPSmI4Vo_t&v?>$~K4y(O~Rb*(MFy_igM7 z*~yYUyR6yQgzWnWMUgDov!!g=lInM+=lOmOk4L`O?{i&qxy&D*_qorRbDwj6?)!ef z#JLd7F6Z2I$S0iYI={rZNk*<{HtIl^mx=h>Cim*04K4+Z4IJtd*-)%6XV2(MCscPiw_a+y*?BKbTS@BZ3AUao^%Zi#PhoY9Vib4N>SE%4>=Jco0v zH_Miey{E;FkdlZSq)e<{`+S3W=*ttvD#hB8w=|2aV*D=yOV}(&p%0LbEWH$&@$X3x~CiF-?ejQ*N+-M zc8zT@3iwkdRT2t(XS`d7`tJQAjRmKAhiw{WOqpuvFp`i@Q@!KMhwKgsA}%@sw8Xo5Y=F zhRJZg)O4uqNWj?V&&vth*H#je6T}}p_<>!Dr#89q@uSjWv~JuW(>FqoJ5^ho0%K?E z9?x_Q;kmcsQ@5=}z@tdljMSt9-Z3xn$k)kEjK|qXS>EfuDmu(Z8|(W?gY6-l z@R_#M8=vxKMAoi&PwnaIYw2COJM@atcgfr=zK1bvjW?9B`-+Voe$Q+H$j!1$Tjn+* z&LY<%)L@;zhnJlB^Og6I&BOR-m?{IW;tyYC%FZ!&Z>kGjHJ6cqM-F z&19n+e1=9AH1VrVeHrIzqlC`w9=*zfmrerF?JMzO&|Mmv;!4DKc(sp+jy^Dx?(8>1 zH&yS_4yL7m&GWX~mdfgH*AB4{CKo;+egw=PrvkTaoBU+P-4u?E|&!c z)DKc;>$$B6u*Zr1SjUh2)FeuWLWHl5TH(UHWkf zLs>7px!c5n;rbe^lO@qlYLzlDVp(z?6rPZel=YB)Uv&n!2{+Mb$-vQl=xKw( zve&>xYx+jW_NJh!FV||r?;hdP*jOXYcLCp>DOtJ?2S^)DkM{{Eb zS$!L$e_o0(^}n3tA1R3-$SNvgBq;DOEo}fNc|tB%%#g4RA3{|euq)p+xd3I8^4E&m zFrD%}nvG^HUAIKe9_{tXB;tl|G<%>yk6R;8L2)KUJw4yHJXUOPM>(-+jxq4R;z8H#>rnJy*)8N+$wA$^F zN+H*3t)eFEgxLw+Nw3};4WV$qj&_D`%ADV2%r zJCPCo%{=z7;`F98(us5JnT(G@sKTZ^;2FVitXyLe-S5(hV&Ium+1pIUB(CZ#h|g)u zSLJJ<@HgrDiA-}V_6B^x1>c9B6%~847JkQ!^KLZ2skm;q*edo;UA)~?SghG8;QbHh z_6M;ouo_1rq9=x$<`Y@EA{C%6-pEV}B(1#sDoe_e1s3^Y>n#1Sw;N|}8D|s|VPd+g z-_$QhCz`vLxxrVMx3ape1xu3*wjx=yKSlM~nFgkNWb4?DDr*!?U)L_VeffF<+!j|b zZ$Wn2$TDv3C3V@BHpSgv3JUif8%hk%OsGZ=OxH@8&4`bbf$`aAMchl^qN>Eyu3JH} z9-S!x8-s4fE=lad%Pkp8hAs~u?|uRnL48O|;*DEU! zuS0{cpk%1E0nc__2%;apFsTm0bKtd&A0~S3Cj^?72-*Owk3V!ZG*PswDfS~}2<8le z5+W^`Y(&R)yVF*tU_s!XMcJS`;(Tr`J0%>p=Z&InR%D3@KEzzI+-2)HK zuoNZ&o=wUC&+*?ofPb0a(E6(<2Amd6%uSu_^-<1?hsxs~0K5^f(LsGqgEF^+0_H=uNk9S0bb!|O8d?m5gQjUKevPaO+*VfSn^2892K~%crWM8+6 z25@V?Y@J<9w%@NXh-2!}SK_(X)O4AM1-WTg>sj1{lj5@=q&dxE^9xng1_z9w9DK>| z6Iybcd0e zyi;Ew!KBRIfGPGytQ6}z}MeXCfLY0?9%RiyagSp_D1?N&c{ zyo>VbJ4Gy`@Fv+5cKgUgs~na$>BV{*em7PU3%lloy_aEovR+J7TfQKh8BJXyL6|P8un-Jnq(ghd!_HEOh$zlv2$~y3krgeH;9zC}V3f`uDtW(%mT#944DQa~^8ZI+zAUu4U(j0YcDfKR$bK#gvn_{JZ>|gZ5+)u?T$w7Q%F^;!Wk?G z(le7r!ufT*cxS}PR6hIVtXa)i`d$-_1KkyBU>qmgz-=T};uxx&sKgv48akIWQ89F{ z0XiY?WM^~;|T8zBOr zs#zuOONzH?svv*jokd5SK8wG>+yMC)LYL|vLqm^PMHcT=`}V$=nIRHe2?h)8WQa6O zPAU}d`1y(>kZiP~Gr=mtJLMu`i<2CspL|q2DqAgAD^7*$xzM`PU4^ga`ilE134XBQ z99P(LhHU@7qvl9Yzg$M`+dlS=x^(m-_3t|h>S}E0bcFMn=C|KamQ)=w2^e)35p`zY zRV8X?d;s^>Cof2SPR&nP3E+-LCkS0J$H!eh8~k0qo$}00b=7!H_I2O+Ro@3O$nPdm ztmbOO^B+IHzQ5w>@@@J4cKw5&^_w6s!s=H%&byAbUtczPQ7}wfTqxxtQNfn*u73Qw zGuWsrky_ajPx-5`R<)6xHf>C(oqGf_Fw|-U*GfS?xLML$kv;h_pZ@Kk$y0X(S+K80 z6^|z)*`5VUkawg}=z`S;VhZhxyDfrE0$(PMurAxl~<>lfZa>JZ288ULK7D` zl9|#L^JL}Y$j*j`0-K6kH#?bRmg#5L3iB4Z)%iF@SqT+Lp|{i`m%R-|ZE94Np7Pa5 zCqC^V3}B(FR340pmF*qaa}M}+h6}mqE~7Sh!9bDv9YRT|>vBNAqv09zXHMlcuhKD| zcjjA(b*XCIwJ33?CB!+;{)vX@9xns_b-VO{i0y?}{!sdXj1GM8+$#v>W7nw;+O_9B z_{4L;C6ol?(?W0<6taGEn1^uG=?Q3i29sE`RfYCaV$3DKc_;?HsL?D_fSYg}SuO5U zOB_f4^vZ_x%o`5|C@9C5+o=mFy@au{s)sKw!UgC&L35aH(sgDxRE2De%(%OT=VUdN ziVLEmdOvJ&5*tCMKRyXctCwQu_RH%;m*$YK&m;jtbdH#Ak~13T1^f89tn`A%QEHWs~jnY~E}p_Z$XC z=?YXLCkzVSK+Id`xZYTegb@W8_baLt-Fq`Tv|=)JPbFsKRm)4UW;yT+J`<)%#ue9DPOkje)YF2fsCilK9MIIK>p*`fkoD5nGfmLwt)!KOT+> zOFq*VZktDDyM3P5UOg`~XL#cbzC}eL%qMB=Q5$d89MKuN#$6|4gx_Jt0Gfn8w&q}%lq4QU%6#jT*MRT% zrLz~C8FYKHawn-EQWN1B75O&quS+Z81(zN)G>~vN8VwC+e+y(`>HcxC{MrJ;H1Z4k zZWuv$w_F0-Ub%MVcpIc){4PGL^I7M{>;hS?;eH!;gmcOE66z3;Z1Phqo(t zVP(Hg6q#0gIKgsg7L7WE!{Y#1nI(45tx2{$34dDd#!Z0NIyrm)HOn5W#7;f4pQci# zDW!FI(g4e668kI9{2+mLwB+=#9bfqgX%!B34V-$wwSN(_cm*^{y0jQtv*4}eO^sOV z*9xoNvX)c9isB}Tgx&ZRjp3kwhTVK?r9;n!x>^XYT z@Q^7zp{rkIs{2mUSE^2!Gf6$6;j~&4=-0cSJJDizZp6LTe8b45;{AKM%v99}{{FfC zz709%u0mC=1KXTo(=TqmZQ;c?$M3z(!xah>aywrj40sc2y3rKFw4jCq+Y+u=CH@_V zxz|qeTwa>+<|H%8Dz5u>ZI5MmjTFwXS-Fv!TDd*`>3{krWoNVx$<133`(ftS?ZPyY z&4@ah^3^i`vL$BZa>O|Nt?ucewzsF)0zX3qmM^|waXr=T0pfIb0*$AwU=?Ipl|1Y; z*Pk6{C-p4MY;j@IJ|DW>QHZQJcp;Z~?8(Q+Kk3^0qJ}SCk^*n4W zu9ZFwLHUx-$6xvaQ)SUQcYd6fF8&x)V`1bIuX@>{mE$b|Yd(qomn3;bPwnDUc0F=; zh*6_((%bqAYQWQ~odER?h>1mkL4kpb3s7`0m@rDKGU*oyF)$j~Ffd4fXV$?`f~rHf zB%Y)@5SXZvfwm10RY5X?TEo)PK_`L6qgBp=#>fO49$D zDq8Ozj0q6213tV5Qq=;fZ0$|KroY{Dz=l@lU^J)?Ko@ti20TRplXzphBi>XGx4bou zEWrkNjz0t5j!_ke{g5I#PUlEU$Km8g8TE|XK=MkU@PT4T><2OVamoK;wJ}3X0L$vX zgd7gNa359*nc)R-0!`2X@FOTB`+oETOPc=ubp5R)VQgY+5BTZZJ2?9QwnO=dnulIUF3gFn;BODC2)65)HeVd%t86sL7Rv^Y+nbn+&l z6BAJY(ETvwI)Ts$aiE8rht4KD*qNyE{8{x6R|%akbTBzw;2+6Echkt+W+`u^XX z_z&x%n '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/test-server/java-v4-server/gradlew.bat b/test-server/java-v4-server/gradlew.bat new file mode 100644 index 00000000..25da30db --- /dev/null +++ b/test-server/java-v4-server/gradlew.bat @@ -0,0 +1,92 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/test-server/java-v4-server/license.txt b/test-server/java-v4-server/license.txt new file mode 100644 index 00000000..2dd564b3 --- /dev/null +++ b/test-server/java-v4-server/license.txt @@ -0,0 +1,4 @@ +/* + * Example file license header. + * File header line two + */ \ No newline at end of file diff --git a/test-server/java-v4-server/s3ec-staging b/test-server/java-v4-server/s3ec-staging new file mode 160000 index 00000000..a95aa3fd --- /dev/null +++ b/test-server/java-v4-server/s3ec-staging @@ -0,0 +1 @@ +Subproject commit a95aa3fddb5abf4e17551c0ef3c247c7a43edf40 diff --git a/test-server/java-v4-server/settings.gradle.kts b/test-server/java-v4-server/settings.gradle.kts new file mode 100644 index 00000000..e7c41714 --- /dev/null +++ b/test-server/java-v4-server/settings.gradle.kts @@ -0,0 +1,19 @@ +/** + * Basic usage of generated server stubs. + */ + +pluginManagement { + val smithyGradleVersion: String by settings + + plugins { + id("software.amazon.smithy.gradle.smithy-base").version(smithyGradleVersion) + } + + repositories { + mavenLocal() + mavenCentral() + gradlePluginPortal() + } +} + +rootProject.name = "S3ECJavaTestServer" diff --git a/test-server/java-v4-server/smithy-build.json b/test-server/java-v4-server/smithy-build.json new file mode 100644 index 00000000..a0fcb8e5 --- /dev/null +++ b/test-server/java-v4-server/smithy-build.json @@ -0,0 +1,11 @@ +{ + "version": "2.0", + "plugins": { + "java-server-codegen": { + "service": "software.amazon.encryption.s3#S3ECTestServer", + "namespace": "software.amazon.encryption.s3", + "headerFile": "license.txt" + } + }, + "sources": ["../model"] +} diff --git a/test-server/java-v4-server/specification b/test-server/java-v4-server/specification new file mode 120000 index 00000000..b173f708 --- /dev/null +++ b/test-server/java-v4-server/specification @@ -0,0 +1 @@ +../specification \ No newline at end of file diff --git a/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java new file mode 100644 index 00000000..23f3a11d --- /dev/null +++ b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java @@ -0,0 +1,219 @@ +package software.amazon.encryption.s3; + +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.encryption.s3.internal.InstructionFileConfig; +import software.amazon.encryption.s3.algorithms.AlgorithmSuite; +import software.amazon.encryption.s3.materials.AesKeyring; +import software.amazon.encryption.s3.materials.Keyring; +import software.amazon.encryption.s3.materials.KmsKeyring; +import software.amazon.encryption.s3.materials.MaterialsDescription; +import software.amazon.encryption.s3.materials.PartialRsaKeyPair; +import software.amazon.encryption.s3.materials.RsaKeyring; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.GenericServerError; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.service.CreateClientOperation; +import software.amazon.smithy.java.server.RequestContext; + +import javax.crypto.spec.SecretKeySpec; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.security.KeyFactory; +import java.security.NoSuchAlgorithmException; +import java.security.PublicKey; +import java.security.interfaces.RSAPrivateCrtKey; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.RSAPublicKeySpec; +import java.security.spec.PKCS8EncodedKeySpec; +import java.util.Map; +import java.util.UUID; + +import static software.amazon.encryption.s3.CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT; +import static software.amazon.encryption.s3.CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT; +import static software.amazon.encryption.s3.CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT; + +public class CreateClientOperationImpl implements CreateClientOperation { + private final Map clientCache_; + private final Map keyringCache_; + + public CreateClientOperationImpl(Map clientCache, Map keyringCache) { + clientCache_ = clientCache; + keyringCache_ = keyringCache; + } + + // Copied from S3EC. + private boolean onlyOneNonNull(Object... values) { + boolean haveOneNonNull = false; + for (Object o : values) { + if (o != null) { + if (haveOneNonNull) { + return false; + } + + haveOneNonNull = true; + } + } + + return haveOneNonNull; + } + + @Override + public CreateClientOutput createClient(CreateClientInput input, RequestContext context) { + try { + KeyMaterial key = input.getConfig().getKeyMaterial(); + if (!onlyOneNonNull(key.getAesKey(), key.getKmsKeyId(), key.getRsaKey())) { + throw new RuntimeException("KeyMaterial must be only one, non-null input!"); + } + Keyring keyring; + if (key.getAesKey() != null) { + byte[] keyBytes = new byte[key.getAesKey().remaining()]; + key.getAesKey().get(keyBytes); + + AesKeyring.Builder aesBuilder = AesKeyring.builder() + .wrappingKey(new SecretKeySpec(keyBytes, "AES")) + .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()); + + // Add materials description if provided + if (key.getMaterialsDescription() != null && !key.getMaterialsDescription().isEmpty()) { + MaterialsDescription.Builder matDescBuilder = MaterialsDescription.builder(); + for (Map.Entry entry : key.getMaterialsDescription().entrySet()) { + matDescBuilder.put(entry.getKey(), entry.getValue()); + } + aesBuilder.materialsDescription(matDescBuilder.build()); + } + + keyring = aesBuilder.build(); + } else if (key.getRsaKey() != null) { + try { + byte[] keyBytes = new byte[key.getRsaKey().remaining()]; + key.getRsaKey().get(keyBytes); + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(keyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + RSAPrivateCrtKey privateKey = (RSAPrivateCrtKey) keyFactory.generatePrivate(keySpec); + RSAPublicKeySpec publicKeySpec = new RSAPublicKeySpec( + privateKey.getModulus(), + privateKey.getPublicExponent() + ); + + // Generate public key + PublicKey publicKey = keyFactory.generatePublic(publicKeySpec); + + RsaKeyring.Builder rsaBuilder = RsaKeyring.builder() + .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) + .wrappingKeyPair(PartialRsaKeyPair.builder() + .publicKey(publicKey) + .privateKey(privateKey).build()); + + // Add materials description if provided + if (key.getMaterialsDescription() != null && !key.getMaterialsDescription().isEmpty()) { + MaterialsDescription.Builder matDescBuilder = MaterialsDescription.builder(); + for (Map.Entry entry : key.getMaterialsDescription().entrySet()) { + matDescBuilder.put(entry.getKey(), entry.getValue()); + } + rsaBuilder.materialsDescription(matDescBuilder.build()); + } + + keyring = rsaBuilder.build(); + } catch (NoSuchAlgorithmException | InvalidKeySpecException nse) { + throw GenericServerError.builder() + .message(nse.getMessage()) + .build(); + } + } else if (key.getKmsKeyId() != null) { + keyring = KmsKeyring.builder() + .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) + .wrappingKeyId(key.getKmsKeyId()) + .build(); + } else { + throw new RuntimeException("No KeyMaterial found!"); + } + + // Configure S3 client with adaptive retry for throttling + RetryPolicy retryPolicy = RetryPolicy.builder() + .numRetries(5) + .throttlingBackoffStrategy(BackoffStrategy.defaultThrottlingStrategy()) + .build(); + + S3Client wrappedClient = S3Client.builder() + .overrideConfiguration(ClientOverrideConfiguration.builder() + .retryPolicy(retryPolicy) + .build()) + .build(); + + // V4-Improved server configuration + S3EncryptionClient.Builder s3ClientBuilder = S3EncryptionClient.builderV4() + .wrappedClient(wrappedClient) + .keyring(keyring) + .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) + .enableLegacyUnauthenticatedModes(input.getConfig().isEnableLegacyUnauthenticatedModes()); + + // Client Creation + boolean instFilePut = false; + if (input.getConfig().getInstructionFileConfig() != null) { + instFilePut = input.getConfig().getInstructionFileConfig().isEnableInstructionFilePutObject(); + s3ClientBuilder.instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(S3Client.create()) + .enableInstructionFilePutObject(instFilePut) + .build()); + } + + // Configure commitment policy if provided + if (input.getConfig().getCommitmentPolicy() != null) { + CommitmentPolicy policy = getCommitmentPolicy(input.getConfig().getCommitmentPolicy()); + s3ClientBuilder.commitmentPolicy(policy); + } + + // Configure encryption algorithm if provided + if (input.getConfig().getEncryptionAlgorithm() != null) { + AlgorithmSuite algorithm = getAlgorithmSuite(input.getConfig().getEncryptionAlgorithm()); + s3ClientBuilder.encryptionAlgorithm(algorithm); + } + + S3Client s3Client = s3ClientBuilder.build(); + + UUID uuid = UUID.randomUUID(); + String uuidString = uuid.toString(); + clientCache_.put(uuidString, s3Client); + keyringCache_.put(uuidString, keyring); + return CreateClientOutput.builder() + .clientId(uuidString) + .build(); + } catch (Exception e) { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw GenericServerError.builder() + .message(stackTrace) + .build(); + } + } + + private static AlgorithmSuite getAlgorithmSuite(EncryptionAlgorithm input) { + if (input.equals(EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF)) { + return AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF; + } else if (input.equals(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF)) { + return AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF; + } else if (input.equals(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY)) { + return AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY; + } else { + throw new RuntimeException("Unknown encryption algorithm: " + input); + } + } + + private static software.amazon.encryption.s3.CommitmentPolicy getCommitmentPolicy(software.amazon.encryption.s3.model.CommitmentPolicy input) { + if (input.equals(software.amazon.encryption.s3.model.CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT)) { + return FORBID_ENCRYPT_ALLOW_DECRYPT; + } else if (input.equals(software.amazon.encryption.s3.model.CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT)) { + return REQUIRE_ENCRYPT_ALLOW_DECRYPT; + } else if (input.equals(software.amazon.encryption.s3.model.CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT)) { + return REQUIRE_ENCRYPT_REQUIRE_DECRYPT; + } else { + throw new RuntimeException("Unknown commitment policy: " + input); + } + } +} diff --git a/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java new file mode 100644 index 00000000..a1964085 --- /dev/null +++ b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java @@ -0,0 +1,86 @@ +package software.amazon.encryption.s3; + +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.encryption.s3.model.GenericServerError; +import software.amazon.encryption.s3.model.GetObjectInput; +import software.amazon.encryption.s3.model.GetObjectOutput; +import software.amazon.encryption.s3.model.S3EncryptionClientError; +import software.amazon.encryption.s3.service.GetObjectOperation; +import software.amazon.smithy.java.server.RequestContext; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; + +import static software.amazon.encryption.s3.MetadataUtils.metadataListToMap; +import static software.amazon.encryption.s3.MetadataUtils.metadataMapToList; +import static software.amazon.encryption.s3.S3EncryptionClient.withAdditionalConfiguration; + +public class GetObjectOperationImpl implements GetObjectOperation { + private final Map clientCache_; + + public GetObjectOperationImpl(Map clientCache) { + clientCache_ = clientCache; + } + + @Override + public GetObjectOutput getObject(GetObjectInput input, RequestContext context) { + try { + S3Client s3Client = clientCache_.get(input.getClientID()); + Map ecMap = metadataListToMap(input.getMetadata()); + + try { + ResponseBytes resp = s3Client.getObjectAsBytes(builder -> { + builder.bucket(input.getBucket()) + .key(input.getKey()); + + // Add custom instruction file suffix if provided + if (input.getInstructionFileSuffix() != null && !input.getInstructionFileSuffix().isEmpty()) { + builder.overrideConfiguration(config -> config + .putExecutionAttribute(S3EncryptionClient.CUSTOM_INSTRUCTION_FILE_SUFFIX, input.getInstructionFileSuffix()) + .applyMutation(c -> withAdditionalConfiguration(ecMap).accept(c))); + } else { + builder.overrideConfiguration(withAdditionalConfiguration(ecMap)); + } + + // Add range header if provided + if (input.getRange() != null && !input.getRange().isEmpty()) { + builder.range(input.getRange()); + } + }); + + List mdAsList = metadataMapToList(resp.response().metadata()); + // Can't use asBB else it gets mad bc cant access backing array + ByteBuffer bb = ByteBuffer.wrap(resp.asByteArray()); + GetObjectOutput output = GetObjectOutput.builder() + .body(bb) + .metadata(mdAsList) + .build(); + return output; + } catch (S3EncryptionClientException s3EncryptionClientException) { + // Modeled exceptions MUST be returned as such + StringWriter sw = new StringWriter(); + s3EncryptionClientException.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw S3EncryptionClientError.builder() + .message(stackTrace) + .build(); + } + } catch (Exception e) { + // Don't wrap modeled errors + if (e instanceof S3EncryptionClientError) { + throw e; + } + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw GenericServerError.builder() + .message(stackTrace) + .build(); + } + } +} diff --git a/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java new file mode 100644 index 00000000..9eba6a3d --- /dev/null +++ b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/MetadataUtils.java @@ -0,0 +1,43 @@ +package software.amazon.encryption.s3; + +import software.amazon.encryption.s3.model.GenericServerError; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class MetadataUtils { + + /** + * Annoyingly, Smithy doesn't provide an interface for map types + * in HTTP headers, so we have to do the serde ourselves + */ + public static List metadataMapToList(Map md) { + List mdAsList = new ArrayList<>(md.size()); + for (Map.Entry keyValue : md.entrySet()) { + mdAsList.add("[" + keyValue.getKey() + "]:[" + keyValue.getValue() + "]"); + } + return mdAsList; + } + + public static Map metadataListToMap(List mdList) { + Map md = new HashMap<>(); + for (String entry : mdList) { + // Split on "]:[" to separate key and value + String[] parts = entry.split("]:\\["); + if (parts.length == 2) { + // Remove remaining brackets from start and end + String key = parts[0].substring(1); + String value = parts[1].substring(0, parts[1].length() - 1); + md.put(key, value); + } else { + throw GenericServerError.builder() + .message("Malformed metadata list entry: " + entry) + .build(); + } + } + return md; + } + +} diff --git a/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java new file mode 100644 index 00000000..d399f13d --- /dev/null +++ b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/PutObjectOperationImpl.java @@ -0,0 +1,52 @@ +package software.amazon.encryption.s3; + +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.encryption.s3.model.GenericServerError; +import software.amazon.encryption.s3.model.PutObjectInput; +import software.amazon.encryption.s3.model.PutObjectOutput; +import software.amazon.encryption.s3.service.PutObjectOperation; +import software.amazon.smithy.java.server.RequestContext; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Map; + +import static software.amazon.encryption.s3.MetadataUtils.metadataListToMap; +import static software.amazon.encryption.s3.S3EncryptionClient.withAdditionalConfiguration; + +public class PutObjectOperationImpl implements PutObjectOperation { + + private final Map clientCache_; + + public PutObjectOperationImpl(Map clientCache) { + clientCache_ = clientCache; + } + + @Override + public PutObjectOutput putObject(PutObjectInput input, RequestContext context) { + try { + final Map metadata = metadataListToMap(input.getMetadata()); + S3Client s3Client = clientCache_.get(input.getClientID()); + s3Client.putObject(builder -> builder + .bucket(input.getBucket()) + .key(input.getKey()) + .overrideConfiguration(withAdditionalConfiguration(metadata)), + RequestBody.fromByteBuffer(input.getBody()) + ); + // The real S3 doesn't provide bucket/key/metadata, so Test doesn't need to either, but we do anyway + return PutObjectOutput.builder() + .bucket(input.getBucket()) + .key(input.getKey()) + .metadata(input.getMetadata()) + .build(); + } catch (Exception e) { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw GenericServerError.builder() + .message(stackTrace) + .build(); + } + } +} diff --git a/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/ReEncryptOperationImpl.java b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/ReEncryptOperationImpl.java new file mode 100644 index 00000000..6a7cd5b6 --- /dev/null +++ b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/ReEncryptOperationImpl.java @@ -0,0 +1,183 @@ +package software.amazon.encryption.s3; + +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.encryption.s3.internal.ReEncryptInstructionFileRequest; +import software.amazon.encryption.s3.internal.ReEncryptInstructionFileResponse; +import software.amazon.encryption.s3.materials.AesKeyring; +import software.amazon.encryption.s3.materials.MaterialsDescription; +import software.amazon.encryption.s3.materials.PartialRsaKeyPair; +import software.amazon.encryption.s3.materials.RawKeyring; +import software.amazon.encryption.s3.materials.RsaKeyring; +import software.amazon.encryption.s3.model.GenericServerError; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.ReEncryptInput; +import software.amazon.encryption.s3.model.ReEncryptOutput; +import software.amazon.encryption.s3.model.S3EncryptionClientError; +import software.amazon.encryption.s3.service.ReEncryptOperation; +import software.amazon.smithy.java.server.RequestContext; + +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.security.KeyFactory; +import java.security.PublicKey; +import java.security.interfaces.RSAPrivateCrtKey; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.RSAPublicKeySpec; +import java.util.HashMap; +import java.util.Map; + +public class ReEncryptOperationImpl implements ReEncryptOperation { + private final Map clientCache_; + + public ReEncryptOperationImpl(Map clientCache) { + clientCache_ = clientCache; + } + + @Override + public ReEncryptOutput reEncrypt(ReEncryptInput input, RequestContext context) { + try { + S3Client s3Client = clientCache_.get(input.getClientID()); + + // Ensure we have an S3EncryptionClient, not just a plain S3Client + if (!(s3Client instanceof S3EncryptionClient)) { + throw new IllegalStateException( + "Client " + input.getClientID() + " is not an S3EncryptionClient"); + } + + S3EncryptionClient s3EncryptionClient = (S3EncryptionClient) s3Client; + + // Create a new keyring from the provided newKeyMaterial + KeyMaterial newKeyMaterial = input.getNewKeyMaterial(); + if (newKeyMaterial == null) { + throw new IllegalStateException( + "newKeyMaterial is required for ReEncrypt operation"); + } + + RawKeyring newKeyring = createKeyringFromMaterial(newKeyMaterial); + + try { + // Build the ReEncryptInstructionFileRequest + ReEncryptInstructionFileRequest.Builder requestBuilder = + ReEncryptInstructionFileRequest.builder() + .bucket(input.getBucket()) + .key(input.getKey()) + .newKeyring(newKeyring); + + // Add optional instruction file suffix if provided + if (input.getInstructionFileSuffix() != null && !input.getInstructionFileSuffix().isEmpty()) { + requestBuilder.instructionFileSuffix(input.getInstructionFileSuffix()); + } + + // Add optional enforceRotation if provided + if (input.isEnforceRotation() != null) { + requestBuilder.enforceRotation(input.isEnforceRotation()); + } + + ReEncryptInstructionFileRequest reEncryptRequest = requestBuilder.build(); + + // Perform the re-encryption + ReEncryptInstructionFileResponse response = + s3EncryptionClient.reEncryptInstructionFile(reEncryptRequest); + + // Build and return the output + return ReEncryptOutput.builder() + .bucket(response.bucket()) + .key(response.key()) + .instructionFileSuffix(response.instructionFileSuffix()) + .enforceRotation(response.enforceRotation()) + .build(); + + } catch (S3EncryptionClientException s3EncryptionClientException) { + // Modeled exceptions MUST be returned as such + StringWriter sw = new StringWriter(); + s3EncryptionClientException.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw S3EncryptionClientError.builder() + .message(stackTrace) + .build(); + } + } catch (Exception e) { + // Don't wrap modeled errors + if (e instanceof S3EncryptionClientError) { + throw e; + } + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw GenericServerError.builder() + .message(stackTrace) + .build(); + } + } + + /** + * Creates a RawKeyring from KeyMaterial. + * The KeyMaterial should have exactly one of: aesKey, rsaKey, or kmsKeyId set. + */ + private RawKeyring createKeyringFromMaterial(KeyMaterial keyMaterial) { + try { + // Get materials description from KeyMaterial if provided + MaterialsDescription materialsDescription = null; + if (keyMaterial.getMaterialsDescription() != null && !keyMaterial.getMaterialsDescription().isEmpty()) { + MaterialsDescription.Builder builder = MaterialsDescription.builder(); + for (Map.Entry entry : keyMaterial.getMaterialsDescription().entrySet()) { + builder.put(entry.getKey(), entry.getValue()); + } + materialsDescription = builder.build(); + } + + // Check for AES key + if (keyMaterial.getAesKey() != null) { + byte[] aesKeyBytes = new byte[keyMaterial.getAesKey().remaining()]; + keyMaterial.getAesKey().get(aesKeyBytes); + SecretKey secretKey = new SecretKeySpec(aesKeyBytes, "AES"); + + AesKeyring.Builder keyringBuilder = AesKeyring.builder() + .wrappingKey(secretKey); + + if (materialsDescription != null) { + keyringBuilder.materialsDescription(materialsDescription); + } + + return keyringBuilder.build(); + } + + // Check for RSA key + if (keyMaterial.getRsaKey() != null) { + byte[] rsaKeyBytes = new byte[keyMaterial.getRsaKey().remaining()]; + keyMaterial.getRsaKey().get(rsaKeyBytes); + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(rsaKeyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + RSAPrivateCrtKey privateKey = (RSAPrivateCrtKey) keyFactory.generatePrivate(keySpec); + + // Derive the public key from the private key + RSAPublicKeySpec publicKeySpec = new RSAPublicKeySpec( + privateKey.getModulus(), + privateKey.getPublicExponent() + ); + PublicKey publicKey = keyFactory.generatePublic(publicKeySpec); + + PartialRsaKeyPair keyPair = PartialRsaKeyPair.builder() + .privateKey(privateKey) + .publicKey(publicKey) + .build(); + + RsaKeyring.Builder keyringBuilder = RsaKeyring.builder() + .wrappingKeyPair(keyPair); + + if (materialsDescription != null) { + keyringBuilder.materialsDescription(materialsDescription); + } + + return keyringBuilder.build(); + } + + throw new IllegalStateException( + "KeyMaterial must have either aesKey or rsaKey set"); + } catch (Exception e) { + throw new IllegalStateException("Failed to create keyring from KeyMaterial: " + e.getMessage(), e); + } + } +} diff --git a/test-server/java-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java similarity index 67% rename from test-server/java-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java rename to test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java index 8ad437f4..88d5b981 100644 --- a/test-server/java-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java +++ b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java @@ -6,16 +6,16 @@ package software.amazon.encryption.s3; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.encryption.s3.service.S3ECTestServer; +import software.amazon.smithy.java.server.Server; import java.net.URI; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; -import software.amazon.smithy.java.server.Server; -import software.amazon.encryption.s3.service.S3ECTestServer; public class S3ECJavaTestServer implements Runnable { - static final URI endpoint = URI.create("http://localhost:8080"); + static final URI endpoint = URI.create("http://localhost:8088"); public static void main(String[] args) { new S3ECJavaTestServer().run(); @@ -27,16 +27,18 @@ public void run() { // Obviously this can get messy in a real service. // Assume that the tests behave and don't induce weird race conditions. Map clientCache = new ConcurrentHashMap<>(); + Map keyringCache = new ConcurrentHashMap<>(); Server server = Server.builder() - .endpoints(endpoint) - .addService( - S3ECTestServer.builder() - .addCreateClientOperation(new CreateClientOperationImpl(clientCache)) - .addGetObjectOperation(new GetObjectOperationImpl(clientCache)) - .addPutObjectOperation(new PutObjectOperationImpl(clientCache)) - .build()) - .build(); + .endpoints(endpoint) + .addService( + S3ECTestServer.builder() + .addCreateClientOperation(new CreateClientOperationImpl(clientCache, keyringCache)) + .addGetObjectOperation(new GetObjectOperationImpl(clientCache)) + .addPutObjectOperation(new PutObjectOperationImpl(clientCache)) + .addReEncryptOperation(new ReEncryptOperationImpl(clientCache)) + .build()) + .build(); System.out.println("Starting server..."); server.start(); try { diff --git a/test-server/model/client.smithy b/test-server/model/client.smithy index 4de56b5b..11f65f57 100644 --- a/test-server/model/client.smithy +++ b/test-server/model/client.smithy @@ -25,7 +25,44 @@ structure CreateClientOutput { structure KeyMaterial { rsaKey: Blob, aesKey: Blob, - kmsKeyId: String + kmsKeyId: String, + /// Optional materials description for keyring differentiation + /// Used to distinguish between different key materials for rotation enforcement + materialsDescription: MaterialsDescriptionMap +} + +/// Map of materials description key-value pairs +map MaterialsDescriptionMap { + key: String, + value: String +} + +enum CommitmentPolicy { + REQUIRE_ENCRYPT_REQUIRE_DECRYPT + REQUIRE_ENCRYPT_ALLOW_DECRYPT + FORBID_ENCRYPT_ALLOW_DECRYPT +} + +enum EncryptionAlgorithm { + ALG_AES_256_CBC_IV16_NO_KDF + ALG_AES_256_GCM_IV12_TAG16_NO_KDF + ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY +} + +structure InstructionFileConfig { + /// This allows specifying a (non-encrypted) client for languages which + /// support this for instruction files. + /// In general, languages do not require specifying a client; + /// they use the usual wrapped client for instruction file operations, + /// so it is fine to leave it null for now. + /// This also requires a way to create non-encrypted clients which we don't have yet. + /// Some languages (Java) do allow a client to be passed specifically for instruction files, + /// so this should be implemented eventually for full coverage, + /// especially if other languages add this feature. Until then, + /// the Java integ tests are sufficient. + clientId: String, + enableInstructionFilePutObject: Boolean = false, + disableInstructionFile: Boolean = false } structure S3ECConfig { @@ -33,5 +70,8 @@ structure S3ECConfig { enableDelayedAuthenticationMode: Boolean = false, enableLegacyWrappingAlgorithms: Boolean = false, setBufferSize: Long, - keyMaterial: KeyMaterial + keyMaterial: KeyMaterial, + commitmentPolicy: CommitmentPolicy, + encryptionAlgorithm: EncryptionAlgorithm, + instructionFileConfig: InstructionFileConfig, } diff --git a/test-server/model/object.smithy b/test-server/model/object.smithy index 623d8ed3..93e78370 100644 --- a/test-server/model/object.smithy +++ b/test-server/model/object.smithy @@ -21,6 +21,7 @@ resource Object { } read: GetObject put: PutObject + operations: [ReEncrypt] } @idempotent @@ -35,6 +36,14 @@ operation PutObject { @required $key + /// Encryption context/materials description to use for this operation. + /// For most SDKs (Java, .NET, etc.), materials description is embedded in the keyring/materials + /// during client creation and this parameter is typically empty/unused. + /// + /// For C++ SDK: Materials description MUST be passed per-operation via this parameter + /// because the C++ SDK's EncryptionMaterials constructor does not accept materials description. + /// Instead, GetObject/PutObject operations accept a contextMap parameter that becomes the + /// materials description. @httpHeader("Content-Metadata") $metadata @@ -72,7 +81,14 @@ operation GetObject { @required $key - /// Should probably be renamed to be EC specific + /// Encryption context/materials description to use for this operation. + /// For most SDKs (Java, .NET, etc.), materials description is embedded in the keyring/materials + /// during client creation and this parameter is typically empty/unused. + /// + /// For C++ SDK: Materials description MUST be passed per-operation via this parameter + /// because the C++ SDK's EncryptionMaterials constructor does not accept materials description. + /// Instead, GetObject/PutObject operations accept a contextMap parameter that becomes the + /// materials description. @httpHeader("Content-Metadata") $metadata @@ -80,7 +96,16 @@ operation GetObject { @required @notProperty clientID: String - } + + @httpHeader("Range") + @notProperty + range: String + + /// Custom instruction file suffix to use when reading instruction files + @httpHeader("InstructionFileSuffix") + @notProperty + instructionFileSuffix: String + } output := for Object { @httpHeader("Content-Metadata") @@ -93,6 +118,54 @@ operation GetObject { } } +@http(method: "POST", uri: "/object/{bucket}/{key}/reencrypt") +operation ReEncrypt { + input := for Object { + @httpLabel + @required + $bucket + + @httpLabel + @required + $key + + @httpHeader("ClientID") + @required + @notProperty + clientID: String + + /// New key material to use for re-encryption + @httpPayload + @required + @notProperty + newKeyMaterial: KeyMaterial + + /// Custom instruction file suffix for RSA keyring re-encryption + @httpHeader("InstructionFileSuffix") + @notProperty + instructionFileSuffix: String + + /// Whether to enforce rotation by verifying the key has changed + @httpHeader("EnforceRotation") + @notProperty + enforceRotation: Boolean + } + + output := { + @required + bucket: String + + @required + key: String + + @notProperty + instructionFileSuffix: String + + @notProperty + enforceRotation: Boolean + } +} + /// Smithy does not know how to serialize a map list ObjectMetadata { member: String diff --git a/test-server/net-v3-transition-server/.duvet/.gitignore b/test-server/net-v3-transition-server/.duvet/.gitignore new file mode 100644 index 00000000..93956e36 --- /dev/null +++ b/test-server/net-v3-transition-server/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ \ No newline at end of file diff --git a/test-server/net-v3-transition-server/.duvet/config.toml b/test-server/net-v3-transition-server/.duvet/config.toml new file mode 100644 index 00000000..416dcfb9 --- /dev/null +++ b/test-server/net-v3-transition-server/.duvet/config.toml @@ -0,0 +1,27 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "**/*.cs" + +# Include required specifications here +[[specification]] +source = "../specification/s3-encryption/client.md" +[[specification]] +source = "../specification/s3-encryption/decryption.md" +[[specification]] +source = "../specification/s3-encryption/encryption.md" +[[specification]] +source = "../specification/s3-encryption/key-commitment.md" +[[specification]] +source = "../specification/s3-encryption/key-derivation.md" +[[specification]] +source = "../specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "../specification/s3-encryption/data-format/metadata-strategy.md" + +[report.html] +enabled = true + +# Enable snapshots to prevent requirement coverage regressions +[report.snapshot] +enabled = false diff --git a/test-server/net-v3-transition-server/.gitignore b/test-server/net-v3-transition-server/.gitignore new file mode 100644 index 00000000..4c20cbc8 --- /dev/null +++ b/test-server/net-v3-transition-server/.gitignore @@ -0,0 +1,44 @@ +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# NuGet Packages +*.nupkg +*.snupkg +packages/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# VS Code +.vscode/ + +# macOS +.DS_Store + +# Temporary files +*.tmp +*.temp diff --git a/test-server/net-v3-transition-server/Controllers/ClientController.cs b/test-server/net-v3-transition-server/Controllers/ClientController.cs new file mode 100644 index 00000000..3deeff61 --- /dev/null +++ b/test-server/net-v3-transition-server/Controllers/ClientController.cs @@ -0,0 +1,133 @@ +using System.Net; +using System.Security.Cryptography; +using System.Text.Json; +using Amazon.Extensions.S3.Encryption; +using Amazon.Extensions.S3.Encryption.Primitives; +using Microsoft.AspNetCore.Mvc; +using NetV3TransitionServer.Models; +using NetV3TransitionServer.Services; + +namespace NetV3TransitionServer.Controllers; + +[ApiController] +[Route("[controller]")] +public class ClientController(IClientCacheService clientCacheService, ILogger logger) : ControllerBase +{ + [HttpPost] + public IActionResult CreateClient([FromBody] ClientRequest request) + { + // Return 501 for not implemented features by the server + if (request.Config.EnableDelayedAuthenticationMode) + return StatusCode(501, new GenericServerError { Message = "EnableDelayedAuthenticationMode not supported" }); + if (request.Config.SetBufferSize.HasValue) + return StatusCode(501, new GenericServerError { Message = "SetBufferSize not supported" }); + + try + { + EncryptionMaterialsV2 encryptionMaterial; + if (request.Config.KeyMaterial.KmsKeyId != null) + { + // The POST request does not contain encryption context. + // However, encryption context is a required field when using KMS. + // So, we are passing empty dictionary. + var encryptionContext = new Dictionary(); + var kmsKeyId = request.Config.KeyMaterial.KmsKeyId; + encryptionMaterial = new EncryptionMaterialsV2(kmsKeyId, KmsType.KmsContext, encryptionContext); + logger.LogInformation( + "[NET-V3-Transitional] Created EncryptionMaterialsV2: KMS={KmsKeyId}", + kmsKeyId); + } + else if (request.Config.KeyMaterial.RsaKey != null) + { + var rsaKeyBytes = request.Config.KeyMaterial.RsaKey; + var rsaKey = RSA.Create(); + rsaKey.ImportPkcs8PrivateKey(new ReadOnlySpan(rsaKeyBytes), out _); + encryptionMaterial = new EncryptionMaterialsV2(rsaKey, AsymmetricAlgorithmType.RsaOaepSha1); + logger.LogInformation( + "Created EncryptionMaterialsV2: RSA"); + } + else if (request.Config.KeyMaterial.AesKey != null) + { + var aesKeyBytes = request.Config.KeyMaterial.AesKey; + var aes = Aes.Create(); + aes.Key = aesKeyBytes; + encryptionMaterial = new EncryptionMaterialsV2(aes, SymmetricAlgorithmType.AesGcm); + logger.LogInformation( + "[NET-V3-Transitional] Created EncryptionMaterialsV2: AES"); + } else + { + return StatusCode(501, new GenericServerError { Message = "Unknown or missing key material!" }); + } + + var enableLegacyUnauthenticatedModes = request.Config.EnableLegacyUnauthenticatedModes; + var enableLegacyWrappingAlgorithms = request.Config.EnableLegacyWrappingAlgorithms; + var commitmentPolicy = MapCommitmentPolicy(request.Config.CommitmentPolicy); + + // SecurityProfile V2AndLegacy can decrypt from legacy S3EC but V2 cannot + var enableLegacyMode = enableLegacyUnauthenticatedModes || enableLegacyWrappingAlgorithms; + var securityProfile = enableLegacyMode ? SecurityProfile.V2AndLegacy : SecurityProfile.V2; + logger.LogInformation("[NET-V3-Transitional] Created securityProfile= {securityProfile}", securityProfile.ToString()); + + var encryptionAlgorithm = MapEncryptionAlgorithm(request.Config.EncryptionAlgorithm); + // var encryptionAlgorithm = commitmentPolicy == Amazon.Extensions.S3.Encryption.CommitmentPolicy.ForbidEncryptAllowDecrypt ? ContentEncryptionAlgorithm.AesGcm : ContentEncryptionAlgorithm.AesGcmWithCommitment; + logger.LogInformation("[NET-V3-Transitional] Created commitmentPolicy= {commitmentPolicy}", commitmentPolicy); + logger.LogInformation("[NET-V3-Transitional] Created encryptionAlgorithm= {encryptionAlgorithm}", encryptionAlgorithm); + + var configuration = new AmazonS3CryptoConfigurationV2(securityProfile, commitmentPolicy, encryptionAlgorithm); + + // Add retry configuration for throttling + configuration.RetryMode = Amazon.Runtime.RequestRetryMode.Adaptive; + configuration.MaxErrorRetry = 5; + + if (request.Config.InstructionFileConfig?.EnableInstructionFilePutObject == true) + { + configuration.StorageMode = CryptoStorageMode.InstructionFile; + logger.LogInformation("[NET-V3-Transitional] Created StorageMode= InstructionFile"); + } + // Create S3 encryption client + var encryptionClient = new AmazonS3EncryptionClientV2(configuration, encryptionMaterial); + // Add to cache and return client ID + var clientId = clientCacheService.AddClient(encryptionClient); + var response = new ClientResponse { ClientId = clientId }; + + logger.LogInformation("[NET-V3-Transitional] Created S3EC client with ID: {clientId}", clientId); + + return new ContentResult + { + Content = JsonSerializer.Serialize(response), + ContentType = "application/json", + StatusCode = 200 + }; + } + catch (Exception ex) + { + logger.LogError(ex, "[NET-V3-Transitional] Failed to create S3EC client"); + return StatusCode(500, new S3EncryptionClientError + { + Message = $"Failed to create client: {ex.Message}" + }); + } + } + + private static Amazon.Extensions.S3.Encryption.CommitmentPolicy MapCommitmentPolicy(Models.CommitmentPolicy? policy) + { + return policy switch + { + Models.CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT => Amazon.Extensions.S3.Encryption.CommitmentPolicy.RequireEncryptRequireDecrypt, + Models.CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT => Amazon.Extensions.S3.Encryption.CommitmentPolicy.RequireEncryptAllowDecrypt, + Models.CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT => Amazon.Extensions.S3.Encryption.CommitmentPolicy.ForbidEncryptAllowDecrypt, + _ => Amazon.Extensions.S3.Encryption.CommitmentPolicy.ForbidEncryptAllowDecrypt + }; + } + + // This is redundant but useful when tests starts sending EncryptionAlgorithm + private static ContentEncryptionAlgorithm MapEncryptionAlgorithm(Models.EncryptionAlgorithm? algorithm) + { + return algorithm switch + { + Models.EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF => ContentEncryptionAlgorithm.AesGcm, + Models.EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY => ContentEncryptionAlgorithm.AesGcmWithCommitment, + _ => ContentEncryptionAlgorithm.AesGcm + }; + } +} diff --git a/test-server/net-v3-transition-server/Controllers/ObjectController.cs b/test-server/net-v3-transition-server/Controllers/ObjectController.cs new file mode 100644 index 00000000..76548815 --- /dev/null +++ b/test-server/net-v3-transition-server/Controllers/ObjectController.cs @@ -0,0 +1,105 @@ +using System.Text.Json; +using Amazon.S3.Model; +using Microsoft.AspNetCore.Mvc; +using NetV3TransitionServer.Models; +using NetV3TransitionServer.Services; + +namespace NetV3TransitionServer.Controllers; + +[ApiController] +[Route("[controller]")] +public class ObjectController(IClientCacheService clientCacheService, ILogger logger) : ControllerBase +{ + [HttpPut("{bucket}/{key}")] + public async Task PutObject(string bucket, string key) + { + logger.LogInformation("Starting PutObject"); + var clientId = Request.Headers["clientId"].FirstOrDefault(); + if (string.IsNullOrEmpty(clientId)) + return BadRequest(new GenericServerError { Message = "[NET-V3-Transitional] ClientID header is required" }); + + var client = clientCacheService.GetClient(clientId); + if (client == null) + return NotFound(new GenericServerError { Message = $"[NET-V3-Transitional] No client found for ClientID: {clientId}" }); + + try + { + // Read raw body data + using var memoryStream = new MemoryStream(); + // Request is the HTTP request this method is currently handling + await Request.Body.CopyToAsync(memoryStream); + var bodyBytes = memoryStream.ToArray(); + + // Create put request + var putRequest = new PutObjectRequest + { + BucketName = bucket, + Key = key, + InputStream = new MemoryStream(bodyBytes) + }; + + await client.PutObjectAsync(putRequest); + + var response = new { bucket, key }; + + logger.LogInformation( + "[NET-V3-Transitional] Put object succeeded for bucket={bucket}, key={key} and clientId = {clientId}", + bucket, key, clientId); + return new ContentResult + { + Content = JsonSerializer.Serialize(response), + ContentType = "application/json", + StatusCode = 200 + }; + } + catch (Exception ex) + { + logger.LogError(ex, "[NET-V3-Transitional] Failed to put object from S3 for bucket={bucket}, key={key}", bucket, key); + return StatusCode(500, new S3EncryptionClientError { Message = $"[NET-V3-Transitional] Failed to put object: {ex.Message}" }); + } + } + + [HttpGet("{bucket}/{key}")] + public async Task GetObject(string bucket, string key) + { + logger.LogInformation("Starting GetObject"); + var clientId = Request.Headers["clientId"].FirstOrDefault(); + if (string.IsNullOrEmpty(clientId)) + return BadRequest(new GenericServerError { Message = "[NET-V3-Transitional] ClientID header is required" }); + + var client = clientCacheService.GetClient(clientId); + if (client == null) + return NotFound(new GenericServerError { Message = $"[NET-V3-Transitional] No client found for ClientID: {clientId}" }); + + try + { + var getRequest = new GetObjectRequest + { + BucketName = bucket, + Key = key + }; + var response = await client.GetObjectAsync(getRequest); + logger.LogInformation("[NET-V3-Transitional] Got object from S3 for bucket={bucket}, key={key}", bucket, key); + // Read response body + using var memoryStream = new MemoryStream(); + await response.ResponseStream.CopyToAsync(memoryStream); + var bodyBytes = memoryStream.ToArray(); + + // Convert metadata to content-metadata header format + var metadataList = response.Metadata.Keys + .Select(metaDataKey => $"{metaDataKey}={response.Metadata[metaDataKey]}") + .ToList(); + var metadataStr = string.Join(",", metadataList); + + // Set response headers + Response.Headers["Content-Metadata"] = metadataStr; + + return File(bodyBytes, "application/octet-stream"); + } + catch (Exception ex) + { + logger.LogError(ex, "[NET-V3-Transitional] Failed to get object from S3 for bucket={bucket}, key={key}", bucket, key); + return StatusCode(500, new S3EncryptionClientError { Message = ex.Message }); + } + } +} \ No newline at end of file diff --git a/test-server/net-v3-transition-server/Makefile b/test-server/net-v3-transition-server/Makefile new file mode 100644 index 00000000..eba78e1c --- /dev/null +++ b/test-server/net-v3-transition-server/Makefile @@ -0,0 +1,43 @@ +# Makefile for S3 Encryption Client .NET Testing + +.PHONY: build-server start-server stop-server wait-for-server + +PID_FILE_NET_V3_TRANSITION := net-v3-transition-server.pid +PORT_NET_V3_TRANSITION := 8100 + +build-server: + @echo "Building .NET V3 transition server..." + dotnet build + +start-server: + $(MAKE) start-net-v3-transition-server + +stop-server: + @echo "Stopping .NET V3 Transition server on port $(PORT_NET_V3_TRANSITION)..." + @lsof -ti:$(PORT_NET_V3_TRANSITION) | xargs kill -9 2>/dev/null || true + @if [ -f $(PID_FILE_NET_V3_TRANSITION) ]; then \ + pkill -P $$(cat $(PID_FILE_NET_V3_TRANSITION)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE_NET_V3_TRANSITION)) 2>/dev/null || true; \ + rm -f $(PID_FILE_NET_V3_TRANSITION); \ + fi + @rm -f server.log + @echo "Server stopped" + +# Start .NET V3 transition server in background +start-net-v3-transition-server: + @echo "Starting .NET V3 transition server..." + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + dotnet run > server.log 2>&1 & echo $$! > $(PID_FILE_NET_V3_TRANSITION) + @echo ".NET V3 transition server starting..." + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT_NET_V3_TRANSITION) + +duvet: + duvet report + +view-report-mac: + open .duvet/reports/report.html diff --git a/test-server/net-v3-transition-server/Models/ClientRequest.cs b/test-server/net-v3-transition-server/Models/ClientRequest.cs new file mode 100644 index 00000000..07fe8520 --- /dev/null +++ b/test-server/net-v3-transition-server/Models/ClientRequest.cs @@ -0,0 +1,55 @@ +using System.ComponentModel.DataAnnotations; +using System.Text.Json.Serialization; + +namespace NetV3TransitionServer.Models; + +public class ClientRequest +{ + [Required] + public ClientConfig Config { get; set; } = new(); +} + +public class ClientConfig +{ + public bool EnableLegacyUnauthenticatedModes { get; set; } = false; + public bool EnableLegacyWrappingAlgorithms { get; set; } = false; + public bool EnableDelayedAuthenticationMode { get; set; } = false; + public long? SetBufferSize { get; set; } + [Required] + public KeyMaterial KeyMaterial { get; set; } = new(); + [JsonPropertyName("commitmentPolicy")] + public CommitmentPolicy? CommitmentPolicy { get; set; } + [JsonPropertyName("encryptionAlgorithm")] + public EncryptionAlgorithm? EncryptionAlgorithm { get; set; } + public InstructionFileConfig? InstructionFileConfig { get; set; } +} + +public class KeyMaterial +{ + public byte[]? RsaKey { get; set; } + public byte[]? AesKey { get; set; } + public string? KmsKeyId { get; set; } +} + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum CommitmentPolicy +{ + REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + REQUIRE_ENCRYPT_ALLOW_DECRYPT, + FORBID_ENCRYPT_ALLOW_DECRYPT +} + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum EncryptionAlgorithm +{ + ALG_AES_256_CBC_IV16_NO_KDF, + ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY +} + +public class InstructionFileConfig +{ + public string? ClientId { get; set; } + public bool EnableInstructionFilePutObject { get; set; } = false; + public bool DisableInstructionFile { get; set; } = false; +} \ No newline at end of file diff --git a/test-server/net-v3-transition-server/Models/ClientResponse.cs b/test-server/net-v3-transition-server/Models/ClientResponse.cs new file mode 100644 index 00000000..43c94a3e --- /dev/null +++ b/test-server/net-v3-transition-server/Models/ClientResponse.cs @@ -0,0 +1,8 @@ +using System.Text.Json.Serialization; + +namespace NetV3TransitionServer.Models; + +public class ClientResponse +{ + [JsonPropertyName("clientId")] public string ClientId { get; set; } = string.Empty; +} \ No newline at end of file diff --git a/test-server/net-v3-transition-server/Models/ErrorModels.cs b/test-server/net-v3-transition-server/Models/ErrorModels.cs new file mode 100644 index 00000000..7fbf6680 --- /dev/null +++ b/test-server/net-v3-transition-server/Models/ErrorModels.cs @@ -0,0 +1,17 @@ +using System.Text.Json.Serialization; + +namespace NetV3TransitionServer.Models; + +public class GenericServerError +{ + [JsonPropertyName("__type")] + public string Type { get; set; } = "software.amazon.encryption.s3#GenericServerError"; + public string Message { get; set; } = string.Empty; +} + +public class S3EncryptionClientError +{ + [JsonPropertyName("__type")] + public string Type { get; set; } = "software.amazon.encryption.s3#S3EncryptionClientError"; + public string Message { get; set; } = string.Empty; +} diff --git a/test-server/net-v3-transition-server/NetV3TransitionServer.csproj b/test-server/net-v3-transition-server/NetV3TransitionServer.csproj new file mode 100644 index 00000000..269f555f --- /dev/null +++ b/test-server/net-v3-transition-server/NetV3TransitionServer.csproj @@ -0,0 +1,27 @@ + + + + net8.0 + enable + enable + false + + + + false + + + + + + + + + + + + + + + + diff --git a/test-server/net-v3-transition-server/Program.cs b/test-server/net-v3-transition-server/Program.cs new file mode 100644 index 00000000..138743c9 --- /dev/null +++ b/test-server/net-v3-transition-server/Program.cs @@ -0,0 +1,17 @@ +using NetV3TransitionServer.Services; + +var builder = WebApplication.CreateBuilder(args); + +builder.Services.AddControllers(); +builder.Services.AddSingleton(); + +const int port = 8100; + +builder.WebHost.UseUrls($"http://localhost:{port}"); + +var app = builder.Build(); + +app.MapControllers(); + +Console.WriteLine($"Starting server on port {port}"); +app.Run(); diff --git a/test-server/net-v3-transition-server/README.md b/test-server/net-v3-transition-server/README.md new file mode 100644 index 00000000..ea925c73 --- /dev/null +++ b/test-server/net-v3-transition-server/README.md @@ -0,0 +1,66 @@ +# Net-V3-Transition-Server + +A .NET test server for Amazon S3 encryption client .NET v3 transition. + +## Project Structure + +``` +net-v3-transition-server/ +├── Controllers/ # API controllers +├── Models/ # Data models +├── Services/ # Business logic services +├── Program.cs # Application entry point +├── NetV3TransitionServer.csproj # Project file +└── README.md # This file +``` + +## Running the Server + +For S3 Encryption Client v3 transition (runs on port 8100): + +```bash +dotnet run +``` + +## API Endpoints + +### Client Management + +- `POST /Client` - Create a new S3 encryption client + +### Object Operations + +- `PUT /{bucket}/{key}` - Upload an encrypted object to S3 +- `GET /{bucket}/{key}` - Download and decrypt an object from S3 + +All object operations require a `clientId` header to specify which client to use. + +## Example Usage + +### Create a Client + +```bash +curl -i -X POST \ + -H "Content-Type: application/json" \ + -H "User-Agent: smithy-java/0.0.3 ua/2.1 os/macos#15.5 lang/java#23.0.2" \ + -d '{"config":{"enableLegacyUnauthenticatedModes":true,"enableLegacyWrappingAlgorithms":true,"keyMaterial":{"kmsKeyId":"arn:aws:kms:us-west-2:370957321024:alias/S3EC-Test-Server-Github-KMS-Key"}, "encryptionContext": {"abc": "b"}, "CommitmentPolicy":"REQUIRE_ENCRYPT_REQUIRE_DECRYPT"}}' \ + http://localhost:8100/client +``` + +### Upload an Object + +```bash +curl -X PUT \ + -H "clientid: 7978763a-a02b-4dea-a5d4-78ef11d13d12" \ + -H "content-type: application/octet-stream" \ + -d "simple-test-input-net" \ + http://localhost:8083/object/s3ec-test-server-github-bucket/cross-lang-test-key-kms-ec-dotnet +``` + +### Download an Object + +```bash +curl -X GET \ + -H "clientid: 7978763a-a02b-4dea-a5d4-78ef11d13d12" \ + http://localhost:8083/object/s3ec-test-server-github-bucket/cross-lang-test-key-kms-ec-dotnet +``` diff --git a/test-server/net-v3-transition-server/Services/ClientCacheService.cs b/test-server/net-v3-transition-server/Services/ClientCacheService.cs new file mode 100644 index 00000000..0e7332ca --- /dev/null +++ b/test-server/net-v3-transition-server/Services/ClientCacheService.cs @@ -0,0 +1,28 @@ +using Amazon.Extensions.S3.Encryption; +using System.Collections.Concurrent; + +namespace NetV3TransitionServer.Services; + +public interface IClientCacheService +{ + string AddClient(AmazonS3EncryptionClientV2 client); + AmazonS3EncryptionClientV2? GetClient(string clientId); +} + +public class ClientCacheService : IClientCacheService +{ + private readonly ConcurrentDictionary _clients = new(); + + public string AddClient(AmazonS3EncryptionClientV2 client) + { + var clientId = Guid.NewGuid().ToString(); + _clients[clientId] = client; + return clientId; + } + + public AmazonS3EncryptionClientV2? GetClient(string clientId) + { + _clients.TryGetValue(clientId, out var client); + return client; + } +} diff --git a/test-server/net-v3-transition-server/s3ec-v3-transition-branch b/test-server/net-v3-transition-server/s3ec-v3-transition-branch new file mode 160000 index 00000000..7a552940 --- /dev/null +++ b/test-server/net-v3-transition-server/s3ec-v3-transition-branch @@ -0,0 +1 @@ +Subproject commit 7a55294068bb3bb7f96226efd6d9edcd1057184b diff --git a/test-server/net-v4-server/.duvet/.gitignore b/test-server/net-v4-server/.duvet/.gitignore new file mode 100644 index 00000000..93956e36 --- /dev/null +++ b/test-server/net-v4-server/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ \ No newline at end of file diff --git a/test-server/net-v4-server/.duvet/config.toml b/test-server/net-v4-server/.duvet/config.toml new file mode 100644 index 00000000..0548b05c --- /dev/null +++ b/test-server/net-v4-server/.duvet/config.toml @@ -0,0 +1,27 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "**/*.cs" + +# Include required specifications here +[[specification]] +source = "../specification/s3-encryption/client.md" +[[specification]] +source = "../specification/s3-encryption/decryption.md" +[[specification]] +source = "../specification/s3-encryption/encryption.md" +[[specification]] +source = "../specification/s3-encryption/key-commitment.md" +[[specification]] +source = "../specification/s3-encryption/key-derivation.md" +[[specification]] +source = "../specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "../specification/s3-encryption/data-format/metadata-strategy.md" + +[report.html] +enabled = true + +# Enable snapshots to prevent requirement coverage regressions +[report.snapshot] +enabled = false \ No newline at end of file diff --git a/test-server/net-v4-server/.gitignore b/test-server/net-v4-server/.gitignore new file mode 100644 index 00000000..4c20cbc8 --- /dev/null +++ b/test-server/net-v4-server/.gitignore @@ -0,0 +1,44 @@ +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# NuGet Packages +*.nupkg +*.snupkg +packages/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# VS Code +.vscode/ + +# macOS +.DS_Store + +# Temporary files +*.tmp +*.temp diff --git a/test-server/net-v4-server/Controllers/ClientController.cs b/test-server/net-v4-server/Controllers/ClientController.cs new file mode 100644 index 00000000..2ef8b921 --- /dev/null +++ b/test-server/net-v4-server/Controllers/ClientController.cs @@ -0,0 +1,144 @@ +using System.Security.Cryptography; +using System.Text.Json; +using Amazon.Extensions.S3.Encryption; +using Amazon.Extensions.S3.Encryption.Primitives; +using Microsoft.AspNetCore.Mvc; +using NetV4Server.Models; +using NetV4Server.Services; + +namespace NetV4Server.Controllers; + +[ApiController] +[Route("[controller]")] +public class ClientController(IClientCacheService clientCacheService, ILogger logger) : ControllerBase +{ + [HttpPost] + public IActionResult CreateClient([FromBody] ClientRequest request) + { + // Return 501 for not implemented features by the server + if (request.Config.EnableDelayedAuthenticationMode ?? false) + return StatusCode(501, new GenericServerError { Message = "[NET-V4] EnableDelayedAuthenticationMode not supported" }); + if (request.Config.SetBufferSize.HasValue) + return StatusCode(501, new GenericServerError { Message = "[NET-V4] SetBufferSize not supported" }); + + try + { + EncryptionMaterialsV4 encryptionMaterial; + if (request.Config.KeyMaterial.KmsKeyId != null) + { + // The POST request does not contain encryption context. + // However, encryption context is a required field when using KMS. + // So, we are passing empty dictionary. + var encryptionContext = new Dictionary(); + var kmsKeyId = request.Config.KeyMaterial.KmsKeyId; + encryptionMaterial = new EncryptionMaterialsV4(kmsKeyId, KmsType.KmsContext, encryptionContext); + logger.LogInformation( + "[NET-V4] Created EncryptionMaterialsV4: KMS={KmsKeyId}", + kmsKeyId); + } + else if (request.Config.KeyMaterial.RsaKey != null) + { + var rsaKeyBytes = request.Config.KeyMaterial.RsaKey; + var rsaKey = RSA.Create(); + rsaKey.ImportPkcs8PrivateKey(new ReadOnlySpan(rsaKeyBytes), out _); + encryptionMaterial = new EncryptionMaterialsV4(rsaKey, AsymmetricAlgorithmType.RsaOaepSha1); + logger.LogInformation( + "[NET-V4] Created EncryptionMaterialsV4: RSA"); + } + else if (request.Config.KeyMaterial.AesKey != null) + { + var aesKeyBytes = request.Config.KeyMaterial.AesKey; + var aes = Aes.Create(); + aes.Key = aesKeyBytes; + encryptionMaterial = new EncryptionMaterialsV4(aes, SymmetricAlgorithmType.AesGcm); + logger.LogInformation( + "[NET-V4] Created EncryptionMaterialsV4: AES"); + } else + { + return StatusCode(501, new GenericServerError { Message = "[NET-V4] Unknown or missing key material!" }); + } + var enableLegacyUnauthenticatedModes = request.Config.EnableLegacyUnauthenticatedModes ?? false; + var enableLegacyWrappingAlgorithms = request.Config.EnableLegacyWrappingAlgorithms ?? false; + var commitmentPolicy = MapCommitmentPolicy(request.Config.CommitmentPolicy); + var isSecurityProfileProvided = request.Config.EnableLegacyUnauthenticatedModes.HasValue || request.Config.EnableLegacyWrappingAlgorithms.HasValue; + var isCommitmentPolicyProvided = request.Config.CommitmentPolicy.HasValue; + var useDefaultConf = !isCommitmentPolicyProvided; + + logger.LogInformation("[NET-V4] isSecurityProfileProvided: {isSecurityProfileProvided}, isCommitmentPolicyProvided: {isCommitmentPolicyProvided}, useDefaultConf: {useDefaultConf}", isSecurityProfileProvided, isCommitmentPolicyProvided, useDefaultConf); + + // SecurityProfile V4AndLegacy can decrypt from legacy S3EC but V4 cannot + var enableLegacyMode = enableLegacyUnauthenticatedModes || enableLegacyWrappingAlgorithms; + var securityProfile = enableLegacyMode ? SecurityProfile.V4AndLegacy : SecurityProfile.V4; + + var encryptionAlgorithm = MapEncryptionAlgorithm(request.Config.EncryptionAlgorithm); + + if (!useDefaultConf) + { + logger.LogInformation("[NET-V4] Created securityProfile= {securityProfile}", securityProfile.ToString()); + logger.LogInformation("[NET-V4] Created commitmentPolicy= {commitmentPolicy}", commitmentPolicy); + logger.LogInformation("[NET-V4] Created encryptionAlgorithm= {encryptionAlgorithm}", encryptionAlgorithm); + } else + { + logger.LogInformation("[NET-V4] Using default configuration for securityProfile, commitmentPolicy and encryptionAlgorithm"); + } + + var configuration = useDefaultConf + ? new AmazonS3CryptoConfigurationV4() + : new AmazonS3CryptoConfigurationV4(securityProfile, commitmentPolicy, encryptionAlgorithm); + + // Add retry configuration for throttling + configuration.RetryMode = Amazon.Runtime.RequestRetryMode.Adaptive; + configuration.MaxErrorRetry = 5; + + if (request.Config.InstructionFileConfig?.EnableInstructionFilePutObject == true) + { + configuration.StorageMode = CryptoStorageMode.InstructionFile; + logger.LogInformation("[NET-V3-Transitional] Created StorageMode= InstructionFile"); + } + + // Create S3 encryption client + var encryptionClient = new AmazonS3EncryptionClientV4(configuration, encryptionMaterial); + // Add to cache and return client ID + var clientId = clientCacheService.AddClient(encryptionClient); + var response = new ClientResponse { ClientId = clientId }; + + logger.LogInformation("[NET-V4] Created S3EC client with ID: {clientId}", clientId); + + return new ContentResult + { + Content = JsonSerializer.Serialize(response), + ContentType = "application/json", + StatusCode = 200 + }; + } + catch (Exception ex) + { + logger.LogError(ex, "[NET-V4] Failed to create S3EC client"); + return StatusCode(500, new S3EncryptionClientError + { + Message = $"[NET-V4] Failed to create client: {ex.Message}" + }); + } + } + + private static Amazon.Extensions.S3.Encryption.CommitmentPolicy MapCommitmentPolicy(Models.CommitmentPolicy? policy) + { + return policy switch + { + Models.CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT => Amazon.Extensions.S3.Encryption.CommitmentPolicy.RequireEncryptRequireDecrypt, + Models.CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT => Amazon.Extensions.S3.Encryption.CommitmentPolicy.RequireEncryptAllowDecrypt, + Models.CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT => Amazon.Extensions.S3.Encryption.CommitmentPolicy.ForbidEncryptAllowDecrypt, + _ => Amazon.Extensions.S3.Encryption.CommitmentPolicy.RequireEncryptRequireDecrypt + }; + } + + private static ContentEncryptionAlgorithm MapEncryptionAlgorithm(Models.EncryptionAlgorithm? algorithm) + { + return algorithm switch + { + Models.EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF => ContentEncryptionAlgorithm.AesGcm, + Models.EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY => ContentEncryptionAlgorithm.AesGcmWithCommitment, + _ => ContentEncryptionAlgorithm.AesGcmWithCommitment + }; + } +} diff --git a/test-server/net-v4-server/Controllers/ObjectController.cs b/test-server/net-v4-server/Controllers/ObjectController.cs new file mode 100644 index 00000000..7ebd8fd1 --- /dev/null +++ b/test-server/net-v4-server/Controllers/ObjectController.cs @@ -0,0 +1,105 @@ +using System.Text.Json; +using Amazon.S3.Model; +using Microsoft.AspNetCore.Mvc; +using NetV4Server.Models; +using NetV4Server.Services; + +namespace NetV4Server.Controllers; + +[ApiController] +[Route("[controller]")] +public class ObjectController(IClientCacheService clientCacheService, ILogger logger) : ControllerBase +{ + [HttpPut("{bucket}/{key}")] + public async Task PutObject(string bucket, string key) + { + logger.LogInformation("Starting PutObject"); + var clientId = Request.Headers["clientId"].FirstOrDefault(); + if (string.IsNullOrEmpty(clientId)) + return BadRequest(new GenericServerError { Message = "[NET-V4] ClientID header is required" }); + + var client = clientCacheService.GetClient(clientId); + if (client == null) + return NotFound(new GenericServerError { Message = $"[NET-V4] No client found for ClientID: {clientId}" }); + + try + { + // Read raw body data + using var memoryStream = new MemoryStream(); + // Request is the HTTP request this method is currently handling + await Request.Body.CopyToAsync(memoryStream); + var bodyBytes = memoryStream.ToArray(); + + // Create put request + var putRequest = new PutObjectRequest + { + BucketName = bucket, + Key = key, + InputStream = new MemoryStream(bodyBytes) + }; + + await client.PutObjectAsync(putRequest); + + var response = new { bucket, key }; + + logger.LogInformation( + "[NET-V4] Put object succeeded for bucket={bucket}, key={key} and clientId = {clientId}", + bucket, key, clientId); + return new ContentResult + { + Content = JsonSerializer.Serialize(response), + ContentType = "application/json", + StatusCode = 200 + }; + } + catch (Exception ex) + { + logger.LogError(ex, "[NET-V4] Failed to put object from S3 for bucket={bucket}, key={key}", bucket, key); + return StatusCode(500, new S3EncryptionClientError { Message = $"Failed to put object: {ex.Message}" }); + } + } + + [HttpGet("{bucket}/{key}")] + public async Task GetObject(string bucket, string key) + { + logger.LogInformation("[NET-V4] Starting GetObject"); + var clientId = Request.Headers["clientId"].FirstOrDefault(); + if (string.IsNullOrEmpty(clientId)) + return BadRequest(new GenericServerError { Message = "[NET-V4] ClientID header is required" }); + + var client = clientCacheService.GetClient(clientId); + if (client == null) + return NotFound(new GenericServerError { Message = $"[NET-V4] No client found for ClientID: {clientId}" }); + + try + { + var getRequest = new GetObjectRequest + { + BucketName = bucket, + Key = key + }; + var response = await client.GetObjectAsync(getRequest); + logger.LogInformation("[NET-V4] Got object from S3 for bucket={bucket}, key={key}", bucket, key); + // Read response body + using var memoryStream = new MemoryStream(); + await response.ResponseStream.CopyToAsync(memoryStream); + var bodyBytes = memoryStream.ToArray(); + + // Convert metadata to content-metadata header format + var metadataList = response.Metadata.Keys + .Select(metaDataKey => $"{metaDataKey}={response.Metadata[metaDataKey]}") + .ToList(); + var metadataStr = string.Join(",", metadataList); + + // Set response headers + Response.Headers["Content-Metadata"] = metadataStr; + + return File(bodyBytes, "application/octet-stream"); + } + catch (Exception ex) + { + logger.LogError(ex, "[NET-V4] Failed to get object from S3 for bucket={bucket}, key={key}", bucket, key); + return StatusCode(500, new S3EncryptionClientError { Message = ex.Message }); + } + } +} \ No newline at end of file diff --git a/test-server/net-v4-server/Makefile b/test-server/net-v4-server/Makefile new file mode 100644 index 00000000..b52bbd49 --- /dev/null +++ b/test-server/net-v4-server/Makefile @@ -0,0 +1,45 @@ +# Makefile for S3 Encryption Client .NET Testing + +.PHONY: start-server stop-server wait-for-server + +PID_FILE_NET_V4 := net-V4-server.pid +PORT_NET_V4 := 8090 + +build-server: + @echo "Building .NET V4 improved server..." + dotnet build + +start-server: + $(MAKE) start-net-V4-server; + +stop-server: + @echo "Stopping .NET V4 Improved server on port $(PORT_NET_V4)..." + @lsof -ti:$(PORT_NET_V4) | xargs kill -9 2>/dev/null || true + @if [ -f $(PID_FILE_NET_V4) ]; then \ + pkill -P $$(cat $(PID_FILE_NET_V4)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE_NET_V4)) 2>/dev/null || true; \ + rm -f $(PID_FILE_NET_V4); \ + fi + @rm -f server.log + @echo "Server stopped" + +# Start .NET V4 server in background +# This builds first into bin/V4 and runs through dll +# to avoid simultaneous dotnet run conflict +start-net-V4-server: + @echo "Starting .NET V4 server..." + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + dotnet run --no-build > server.log 2>&1 & echo $! > $(PID_FILE_NET_V4) + @echo ".NET V4 server starting..." + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT_NET_V4) + +duvet: + duvet report + +view-report-mac: + open .duvet/reports/report.html diff --git a/test-server/net-v4-server/Models/ClientRequest.cs b/test-server/net-v4-server/Models/ClientRequest.cs new file mode 100644 index 00000000..76623b9d --- /dev/null +++ b/test-server/net-v4-server/Models/ClientRequest.cs @@ -0,0 +1,55 @@ +using System.ComponentModel.DataAnnotations; +using System.Text.Json.Serialization; + +namespace NetV4Server.Models; + +public class ClientRequest +{ + [Required] + public ClientConfig Config { get; set; } = new(); +} + +public class ClientConfig +{ + public bool? EnableLegacyUnauthenticatedModes { get; set; } + public bool? EnableLegacyWrappingAlgorithms { get; set; } + public bool? EnableDelayedAuthenticationMode { get; set; } + public long? SetBufferSize { get; set; } + [Required] + public KeyMaterial KeyMaterial { get; set; } = new(); + [JsonPropertyName("commitmentPolicy")] + public CommitmentPolicy? CommitmentPolicy { get; set; } + [JsonPropertyName("encryptionAlgorithm")] + public EncryptionAlgorithm? EncryptionAlgorithm { get; set; } + public InstructionFileConfig? InstructionFileConfig { get; set; } +} + +public class KeyMaterial +{ + public byte[]? RsaKey { get; set; } + public byte[]? AesKey { get; set; } + public string? KmsKeyId { get; set; } +} + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum CommitmentPolicy +{ + REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + REQUIRE_ENCRYPT_ALLOW_DECRYPT, + FORBID_ENCRYPT_ALLOW_DECRYPT +} + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum EncryptionAlgorithm +{ + ALG_AES_256_CBC_IV16_NO_KDF, + ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY +} + +public class InstructionFileConfig +{ + public string? ClientId { get; set; } + public bool EnableInstructionFilePutObject { get; set; } = false; + public bool DisableInstructionFile { get; set; } = false; +} \ No newline at end of file diff --git a/test-server/net-v4-server/Models/ClientResponse.cs b/test-server/net-v4-server/Models/ClientResponse.cs new file mode 100644 index 00000000..b4dbb494 --- /dev/null +++ b/test-server/net-v4-server/Models/ClientResponse.cs @@ -0,0 +1,8 @@ +using System.Text.Json.Serialization; + +namespace NetV4Server.Models; + +public class ClientResponse +{ + [JsonPropertyName("clientId")] public string ClientId { get; set; } = string.Empty; +} \ No newline at end of file diff --git a/test-server/net-v4-server/Models/ErrorModels.cs b/test-server/net-v4-server/Models/ErrorModels.cs new file mode 100644 index 00000000..e4b818e3 --- /dev/null +++ b/test-server/net-v4-server/Models/ErrorModels.cs @@ -0,0 +1,17 @@ +using System.Text.Json.Serialization; + +namespace NetV4Server.Models; + +public class GenericServerError +{ + [JsonPropertyName("__type")] + public string Type { get; set; } = "software.amazon.encryption.s3#GenericServerError"; + public string Message { get; set; } = string.Empty; +} + +public class S3EncryptionClientError +{ + [JsonPropertyName("__type")] + public string Type { get; set; } = "software.amazon.encryption.s3#S3EncryptionClientError"; + public string Message { get; set; } = string.Empty; +} diff --git a/test-server/net-v4-server/NetV4Server.csproj b/test-server/net-v4-server/NetV4Server.csproj new file mode 100644 index 00000000..28ddba06 --- /dev/null +++ b/test-server/net-v4-server/NetV4Server.csproj @@ -0,0 +1,28 @@ + + + + net8.0 + enable + enable + false + NetV2V3Server + + + + false + + + + + + + + + + + + + + + + diff --git a/test-server/net-v4-server/Program.cs b/test-server/net-v4-server/Program.cs new file mode 100644 index 00000000..23cf79d9 --- /dev/null +++ b/test-server/net-v4-server/Program.cs @@ -0,0 +1,17 @@ +using NetV4Server.Services; + +var builder = WebApplication.CreateBuilder(args); + +builder.Services.AddControllers(); +builder.Services.AddSingleton(); + +const int port = 8090; + +builder.WebHost.UseUrls($"http://localhost:{port}"); + +var app = builder.Build(); + +app.MapControllers(); + +Console.WriteLine($"Starting server on port {port}"); +app.Run(); diff --git a/test-server/net-v4-server/README.md b/test-server/net-v4-server/README.md new file mode 100644 index 00000000..487d8471 --- /dev/null +++ b/test-server/net-v4-server/README.md @@ -0,0 +1,72 @@ +# Net-V4-Server + +A .NET test server for Amazon S3 encryption client .NET v4. + +## Project Structure + +``` +net-v4-server/ +├── Controllers/ # API controllers +├── Models/ # Data models +├── Services/ # Business logic services +├── Program.cs # Application entry point +├── NetV2V3Server.csproj # Project file +└── README.md # This file +``` + +## Running the Server + +For S3 Encryption Client v2 (runs on port 8083): + +```bash +dotnet run -p:S3EncryptionVersion=v2 +``` + +For S3 Encryption Client v3 (runs on port 8084): + +```bash +dotnet run -p:S3EncryptionVersion=v3 +``` + +## API Endpoints + +### Client Management + +- `POST /Client` - Create a new S3 encryption client + +### Object Operations + +- `PUT /{bucket}/{key}` - Upload an encrypted object to S3 +- `GET /{bucket}/{key}` - Download and decrypt an object from S3 + +All object operations require a `clientId` header to specify which client to use. + +## Example Usage + +### Create a Client + +```bash +curl -i -X POST \ + -H "Content-Type: application/json" \ + -H "User-Agent: smithy-java/0.0.3 ua/2.1 os/macos#15.5 lang/java#23.0.2" \ + -d '{"config":{"keyMaterial":{"kmsKeyId":"arn:aws:kms:us-west-2:370957321024:alias/S3EC-Test-Server-Github-KMS-Key"}, "encryptionContext": {"abc": "b"}, "CommitmentPolicy":"FORBID_ENCRYPT_ALLOW_DECRYPT"}}' \ + http://localhost:8090/client +``` + +### Upload an Object + +```bash +curl -X PUT \ + -H "clientid: 7978763a-a02b-4dea-a5d4-78ef11d13d12" \ + -H "content-type: application/octet-stream" \ + -d "simple-test-input-net" \ + http://localhost:8083/object/s3ec-test-server-github-bucket/cross-lang-test-key-kms-ec-dotnet +``` + +### Download an Object + +```bash +curl -X GET \ + -H "clientid: 7978763a-a02b-4dea-a5d4-78ef11d13d12" \ + http://localhost:8083/object/s3ec-test-server-github-bucket/cross-lang-test-key-kms-ec-dotnet +``` diff --git a/test-server/net-v4-server/Services/ClientCacheService.cs b/test-server/net-v4-server/Services/ClientCacheService.cs new file mode 100644 index 00000000..55764152 --- /dev/null +++ b/test-server/net-v4-server/Services/ClientCacheService.cs @@ -0,0 +1,28 @@ +using Amazon.Extensions.S3.Encryption; +using System.Collections.Concurrent; + +namespace NetV4Server.Services; + +public interface IClientCacheService +{ + string AddClient(AmazonS3EncryptionClientV4 client); + AmazonS3EncryptionClientV4? GetClient(string clientId); +} + +public class ClientCacheService : IClientCacheService +{ + private readonly ConcurrentDictionary _clients = new(); + + public string AddClient(AmazonS3EncryptionClientV4 client) + { + var clientId = Guid.NewGuid().ToString(); + _clients[clientId] = client; + return clientId; + } + + public AmazonS3EncryptionClientV4? GetClient(string clientId) + { + _clients.TryGetValue(clientId, out var client); + return client; + } +} diff --git a/test-server/net-v4-server/s3ec-net-v4-improved b/test-server/net-v4-server/s3ec-net-v4-improved new file mode 160000 index 00000000..9b628b06 --- /dev/null +++ b/test-server/net-v4-server/s3ec-net-v4-improved @@ -0,0 +1 @@ +Subproject commit 9b628b06e5c1bf12696c752afb2631c38cae11f9 diff --git a/test-server/php-v2-transition-server/.duvet/.gitignore b/test-server/php-v2-transition-server/.duvet/.gitignore new file mode 100644 index 00000000..93956e36 --- /dev/null +++ b/test-server/php-v2-transition-server/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ \ No newline at end of file diff --git a/test-server/php-v2-transition-server/.duvet/config.toml b/test-server/php-v2-transition-server/.duvet/config.toml new file mode 100644 index 00000000..64b00927 --- /dev/null +++ b/test-server/php-v2-transition-server/.duvet/config.toml @@ -0,0 +1,24 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "local-php-sdk/src/S3/**/*.php" + +[[source]] +pattern = "local-php-sdk/src/Crypto/**/*.php" + +# Include required specifications here +[[specification]] +source = "../specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "../specification/s3-encryption/data-format/metadata-strategy.md" +[[specification]] +source = "../specification/s3-encryption/encryption.md" +[[specification]] +source = "../specification/s3-encryption/key-derivation.md" + +[report.html] +enabled = true + +# Enable snapshots to prevent requirement coverage regressions +[report.snapshot] +enabled = false diff --git a/test-server/php-v2-transition-server/.gitignore b/test-server/php-v2-transition-server/.gitignore new file mode 100644 index 00000000..07108589 --- /dev/null +++ b/test-server/php-v2-transition-server/.gitignore @@ -0,0 +1,4 @@ +vendor/* +cookies.txt +server.pid +composer.lock \ No newline at end of file diff --git a/test-server/php-v2-transition-server/Makefile b/test-server/php-v2-transition-server/Makefile new file mode 100644 index 00000000..a3d038de --- /dev/null +++ b/test-server/php-v2-transition-server/Makefile @@ -0,0 +1,39 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: build-server start-server stop-server wait-for-server + +PID_FILE := server.pid +PORT := 8099 + +build-server: + @echo "Building PHP V2 Transition server..." + composer install + +start-server: + @echo "Starting PHP V2 Transition server..." + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + composer run start --timeout=0 > server.log 2>&1 & echo $$! > $(PID_FILE) + @echo "PHP V2 Transition server starting..." + +stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true + @if [ -f $(PID_FILE) ]; then \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ + fi + @rm -f server.log + @echo "Server stopped" + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT) + +duvet: + duvet report + +view-report-mac: + open .duvet/reports/report.html diff --git a/test-server/php-v2-transition-server/composer.json b/test-server/php-v2-transition-server/composer.json new file mode 100644 index 00000000..6a0f263b --- /dev/null +++ b/test-server/php-v2-transition-server/composer.json @@ -0,0 +1,36 @@ +{ + "name": "aws/s3ec-php-v2-transition-test-server", + "description": "PHP V2 Transition implementation of the S3EC Test Server framework", + "type": "project", + "license": "Apache-2.0", + "repositories": [ + { + "type": "path", + "url": "./local-php-sdk", + "options": { + "symlink": true + } + } + ], + "require": { + "php": ">=7.4", + "aws/aws-sdk-php": "@dev", + "ramsey/uuid": "^4.9" + }, + "autoload": { + "psr-4": { + "S3EC\\PhpV2Server\\": "src/" + } + }, + "scripts": { + "start": [ + "php -S 0.0.0.0:8099 src/index.php" + ] + }, + "config": { + "optimize-autoloader": true, + "platform": { + "php": "8.1" + } + } +} \ No newline at end of file diff --git a/test-server/php-v2-transition-server/local-php-sdk b/test-server/php-v2-transition-server/local-php-sdk new file mode 120000 index 00000000..4610ddb9 --- /dev/null +++ b/test-server/php-v2-transition-server/local-php-sdk @@ -0,0 +1 @@ +../php-v3-server/local-php-sdk \ No newline at end of file diff --git a/test-server/php-v2-transition-server/src/client.php b/test-server/php-v2-transition-server/src/client.php new file mode 100644 index 00000000..534d47a7 --- /dev/null +++ b/test-server/php-v2-transition-server/src/client.php @@ -0,0 +1,82 @@ +toString(); + $kmsKeyId = $keyMaterial["kmsKeyId"] ?? null; + $commitmentPolicy = $configData['commitmentPolicy'] ?? "FORBID_ENCRYPT_ALLOW_DECRYPT"; + $instFileConfig = $configData['instructionFileConfig'] ?? null; + $instFilePut = false; + if ($instFileConfig != null) { + $instFilePut = $instFileConfig['enableInstructionFilePutObject'] ?? false; + } + + if ($configData == []) { + return GenericServerError("Invalid config in request body", 400); + } + if (($keyMaterial || $kmsKeyId) === null) { + return GenericServerError("Invalid keyMaterial in config", 400); + } + if ($commitmentPolicy !== "FORBID_ENCRYPT_ALLOW_DECRYPT") { + return GenericServerError( + "Transition server only supports FORBID_ENCRYPT_ALLOW_DECRYPT" + . "commitment policy but received {$commitmentPolicy}" + ); + } + + // Store client configuration instead of objects (AWS objects can't be serialized) + $_SESSION['s3ecCache'][$clientId] = [ + 's3Config' => [ + 'region' => 'us-west-2', + 'version' => 'latest', + 'http' => [ + 'debug' => false, + 'verify' => true, + 'curl' => [ + CURLOPT_VERBOSE => false, + CURLOPT_NOPROGRESS => true + ] + ] + ], + 'kmsConfig' => [ + 'region' => 'us-west-2', + 'version' => 'latest', + 'http' => [ + 'debug' => false, + 'verify' => true, + 'curl' => [ + CURLOPT_VERBOSE => false, + CURLOPT_NOPROGRESS => true + ] + ] + ], + 'kmsKeyId' => $kmsKeyId, + 'legacy' => $legacyAlgorithms, + 'commitmentPolicy' => $commitmentPolicy, + 'instFilePut' => $instFilePut, + 'created' => time() + ]; + + // Auto-update cookies.txt with current session ID so tests can access cached clients + writeSessionIdToCookiesFile(session_id()); + + header("Content-Type: application/json"); + return json_encode([ + 'clientId' => $clientId, + ]); +} diff --git a/test-server/php-v2-transition-server/src/errors.php b/test-server/php-v2-transition-server/src/errors.php new file mode 100644 index 00000000..67449c11 --- /dev/null +++ b/test-server/php-v2-transition-server/src/errors.php @@ -0,0 +1,42 @@ + 'GenericServerError', + 'message' => $message + ]; + + return json_encode($errorResponse); +} + +/** + * Used for modeled errors, e.g. errors thrown by the S3EC + * Tests SHOULD expect this error in negative tests. + * + * @param string $message The error message to include in the response + * @return string JSON-encoded error response + */ +function S3EncryptionClientError($message) +{ + http_response_code(500); + header('Content-Type: application/json'); + + $errorResponse = [ + "__type" => "software.amazon.encryption.s3#S3EncryptionClientError", + 'message' => $message + ]; + + return json_encode($errorResponse); +} diff --git a/test-server/php-v2-transition-server/src/get_object.php b/test-server/php-v2-transition-server/src/get_object.php new file mode 100644 index 00000000..656a337a --- /dev/null +++ b/test-server/php-v2-transition-server/src/get_object.php @@ -0,0 +1,104 @@ + $legacy, + '@MaterialsProvider' => $materialProvider, + '@KmsEncryptionContext' => $encryptionContext, + '@CommitmentPolicy' => $commitmentPolicy, + 'Bucket' => $bucket, + 'Key' => $key, + ]; + + // Add custom instruction file suffix if provided + if (!is_null($instructionFileSuffix) && !empty($instructionFileSuffix)) { + $getObjectParams['@InstructionFileSuffix'] = $instructionFileSuffix; + } + + $result = $s3ec->getObject($getObjectParams); + + // Capture and discard any unwanted output from AWS SDK + $unwantedOutput = ob_get_clean(); + if (!empty($unwantedOutput)) { + error_log("AWS SDK produced unexpected output: " . strlen($unwantedOutput) . " bytes"); + } + + $body = $result['Body']->getContents(); + $formattedMetadata = formatMetadataForResponse($result["Metadata"]); + + // Now set headers safely + header("Content-Metadata: " . $formattedMetadata); + header("Content-Type: application/octet-stream"); + header("Content-Length: " . strlen($body)); + return $body; + } catch (InvalidArgumentException $e) { + // Clean up output buffer if still active + if (ob_get_level()) { + ob_end_clean(); + } + return GenericServerError("Invalid argument: " . $e->getMessage(), 400); + } catch (Exception $e) { + // Clean up output buffer if still active + if (ob_get_level()) { + ob_end_clean(); + } + if (strpos($e->getMessage(), "@SecurityProfile=V2") !== false) { + return S3EncryptionClientError($e->getMessage() . " " . "Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms"); + } elseif (strpos($e->getMessage(), "One or more reserved keys found in Instruction file when they should not be present.") !== false) { + return S3EncryptionClientError($e->getMessage()); + } elseif (strpos($e->getMessage(), "Expected a V3 envelope but was unable to constuct one.") !== false) { + return S3EncryptionClientError($e->getMessage()); + } elseif (strpos($e->getMessage(), "Malformed metadata envelope.") !== false) { + return S3EncryptionClientError($e->getMessage()); + } else { + error_log("This is the error: " . $e->getMessage()); + return GenericServerError("Server error: " . $e->getMessage(), 500); + } + } +} diff --git a/test-server/php-v2-transition-server/src/index.php b/test-server/php-v2-transition-server/src/index.php new file mode 100644 index 00000000..167834e0 --- /dev/null +++ b/test-server/php-v2-transition-server/src/index.php @@ -0,0 +1,295 @@ += 7 && $parts[5] === 'PHPSESSID') { + error_log("Found session ID in cookies.txt: " . $parts[6]); + return $parts[6]; // Return the session ID value + } + } + + error_log("No PHPSESSID found in cookies.txt file"); + return null; +} + +// Function to write session ID to cookies.txt file +function writeSessionIdToCookiesFile($sessionId) +{ + $cookiesFile = __DIR__ . '/../cookies.txt'; + + // Create Netscape cookie format entry + $cookieLine = "localhost\tFALSE\t/\tFALSE\t0\tPHPSESSID\t$sessionId"; + + // Write header and cookie entry + $content = "# Netscape HTTP Cookie File\n"; + $content .= "# https://curl.se/docs/http-cookies.html\n"; + $content .= "# This file was generated by libcurl! Edit at your own risk.\n\n"; + $content .= $cookieLine . "\n"; + + $result = file_put_contents($cookiesFile, $content); + + if ($result === false) { + error_log("Failed to write session ID to cookies.txt file: $cookiesFile"); + return false; + } + + error_log("Successfully wrote session ID to cookies.txt: $sessionId"); + return true; +} + +set_time_limit(600); +// Start session to persist cache across requests +// First try to use session ID from cookies.txt if available +$sessionId = getSessionIdFromCookiesFile(); +if ($sessionId) { + session_id($sessionId); +} +session_start(); + +// Initialize session cache if it doesn't exist +if (!isset($_SESSION['s3ecCache'])) { + $_SESSION['s3ecCache'] = []; +} + +// Simple router class +class SimpleRouter +{ + private $routes = []; + + public function addRoute($method, $path, $handler) + { + $this->routes[] = [ + 'method' => strtoupper($method), + 'path' => $path, + 'handler' => $handler + ]; + } + + public function handleRequest() + { + $method = $_SERVER['REQUEST_METHOD']; + $path = parse_url($_SERVER['REQUEST_URI'], PHP_URL_PATH); + + foreach ($this->routes as $route) { + if ($route['method'] === $method) { + $params = $this->matchPathWithParams($route['path'], $path); + if ($params !== false) { + return call_user_func($route['handler'], $params); + } + } + } + + // Default 404 response + http_response_code(404); + return json_encode(['error' => 'Not Found']); + } + + private function matchPathWithParams($routePath, $requestPath) + { + // Handle exact matches first (for routes without parameters) + if ($routePath === $requestPath) { + return []; + } + + // Convert route path like '/object/{bucket}/{key}' to regex + $pattern = preg_replace('/\{([^}]+)\}/', '([^/]+)', $routePath); + $pattern = '/^' . str_replace('/', '\/', $pattern) . '$/'; + + if (preg_match($pattern, $requestPath, $matches)) { + array_shift($matches); // Remove full match + + // Extract parameter names + preg_match_all('/\{([^}]+)\}/', $routePath, $paramNames); + $params = []; + + foreach ($paramNames[1] as $index => $paramName) { + $params[$paramName] = $matches[$index] ?? null; + } + + return $params; + } + + return false; + } +} + +// Helper function to get cached client by ID +function getCachedClient($clientId) +{ + if (!isset($_SESSION['s3ecCache'][$clientId])) { + return null; + } + + $config = $_SESSION['s3ecCache'][$clientId]; + + // Recreate the AWS clients from stored configuration + $s3Client = new S3Client($config['s3Config']); + $encryptionClient = new S3EncryptionClientV2($s3Client); + + $kmsClient = new KmsClient($config['kmsConfig']); + $materialsProvider = new KmsMaterialsProviderV2($kmsClient, $config['kmsKeyId']); + + return [ + 's3Client' => $s3Client, + 'encryptionClient' => $encryptionClient, + 'materialsProvider' => $materialsProvider, + 'config' => $config + ]; +} + +function createDefaultClientTuple(): array +{ + $s3Client = new S3Client([ + 'region' => 'us-west-2', + 'version' => 'latest', + 'http' => [ + 'debug' => false, + 'verify' => true, + 'curl' => [ + CURLOPT_VERBOSE => false, + CURLOPT_NOPROGRESS => true + ] + ] + ]); + $encryptionClient = new S3EncryptionClientV2($s3Client); + + $kmsClient = new KmsClient([ + 'region' => 'us-west-2', + 'version' => 'latest', + 'http' => [ + 'debug' => false, + 'verify' => true, + 'curl' => [ + CURLOPT_VERBOSE => false, + CURLOPT_NOPROGRESS => true + ] + ] + ]); + $materialsProvider = new KmsMaterialsProviderV2($kmsClient, 'arn:aws:kms:us-west-2:370957321024:alias/S3EC-Test-Server-Github-KMS-Key'); + + return [ + 'encryptionClient' => $encryptionClient, + 'materialsProvider' => $materialsProvider + ]; +} + +function metadataStringToMap($metadata): array +{ + $md = []; + + if (empty($metadata)) { + return $md; + } + + $mdList = explode(',', $metadata); + + foreach ($mdList as $entry) { + $parts = explode(']:[', $entry); + + if (count($parts) === 2) { + $key = substr($parts[0], 1); + $value = substr($parts[1], 0, -1); + $md[$key] = $value; + } else { + throw new InvalidArgumentException("Malformed metadata list entry: " . $entry); + } + } + + return $md; +} +function formatMetadataForResponse($metadata) +{ + $metadataList = []; + // Handle different metadata input types + if (is_array($metadata)) { + // If it's an associative array (like Python dict) + foreach ($metadata as $key => $value) { + $metadataList[] = $key . '=' . $value; + } + } elseif (is_string($metadata) && !empty($metadata)) { + // If it's already a string, assume it's in the correct format + return $metadata; + } + + // Convert array to comma-separated string + return implode(',', $metadataList); +} + +// Initialize router +$router = new SimpleRouter(); + +// Add basic routes +$router->addRoute('GET', '/', function () { + return json_encode([ + 'service' => 'S3EC PHP v2 Test Server', + 'status' => 'running', + 'port' => 8087, + 'endpoints' => [ + 'GET /' => 'Server status', + 'POST /client' => 'Create an S3EncryptionClient and cache it.', + 'GET /object/{bucket}/{key}' => 'Handle GET requests to /object/{bucket}/{key} by using the S3EncryptionClient to make a GetObject request to S3.', + 'PUT /object/{bucket}/{key}' => 'Handle PUT requests to /object/{bucket}/{key} by using the S3EncryptionClient to make a PutObject request to S3.', + ] + ]); +}); + +$router->addRoute('GET', '/cache', function () { + return json_encode([ + 'sessionId' => session_id(), + 'sessionStatus' => session_status(), + 'totalCachedClients' => count($_SESSION['s3ecCache'] ?? []), + 'allClientIds' => array_keys($_SESSION['s3ecCache'] ?? []), + 'cacheDetails' => $_SESSION['s3ecCache'] ?? [] + ]); +}); + +$router->addRoute('GET', '/object/{bucket}/{key}', function ($params) { + return handleGetObject($params); +}); + +$router->addRoute('PUT', '/object/{bucket}/{key}', function ($params) { + return handlePutObject($params); +}); + +$router->addRoute('POST', '/client', function () { + return handleCreateClient(); +}); + +// Handle the request and output response +$result = $router->handleRequest(); +if ($result !== false) { + echo $result; +} diff --git a/test-server/php-v2-transition-server/src/put_object.php b/test-server/php-v2-transition-server/src/put_object.php new file mode 100644 index 00000000..405257cc --- /dev/null +++ b/test-server/php-v2-transition-server/src/put_object.php @@ -0,0 +1,79 @@ + 'gcm', + 'KeySize' => 256, + ]; + $legacyConfig = $s3ecClientTuple["legacy"] ?? false; + $legacy = null; + if ($legacyConfig === false) { + $legacy = "V2"; + } else { + $legacy = "V2_AND_LEGACY"; + } + $strategy = $s3ecClientTuple["config"]["instFilePut"] ? + new InstructionFileMetadataStrategy($s3Client) : + new HeadersMetadataStrategy(); + try { + $result = $s3ec->putObject([ + '@SecurityProfile' => $legacy, + '@MaterialsProvider' => $materialProvider, + '@KmsEncryptionContext' => $encryptionContext, + '@MetadataStrategy' => $strategy, + '@CipherOptions' => $cipherOptions, + 'Bucket' => $bucket, + 'Key' => $key, + 'Body' => $rawBody, + ]); + + header("Content-Type: application/json"); + return json_encode([ + "bucket" => $bucket, + "key" => $key, + // php for some reason blows java's heap if we pass the metadata + // "metadata" => $encryptionContext + ]); + + } catch (InvalidArgumentException $e) { + return S3EncryptionClientError("Invalid argument: " . $e->getMessage()); + } catch (Exception $e) { + return GenericServerError("Server error: " . $e->getMessage()); + } +} diff --git a/test-server/php-v3-server/.duvet/.gitignore b/test-server/php-v3-server/.duvet/.gitignore new file mode 100644 index 00000000..93956e36 --- /dev/null +++ b/test-server/php-v3-server/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ \ No newline at end of file diff --git a/test-server/php-v3-server/.duvet/config.toml b/test-server/php-v3-server/.duvet/config.toml new file mode 100644 index 00000000..d7627473 --- /dev/null +++ b/test-server/php-v3-server/.duvet/config.toml @@ -0,0 +1,39 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "local-php-sdk/src/S3/**/*.php" + +[[source]] +pattern = "local-php-sdk/src/Crypto/**/*.php" + +[[source]] +pattern = "local-php-sdk/tests/S3/**/*.php" + +[[source]] +pattern = "local-php-sdk/tests/Crypto/**/*.php" + +[[source]] +pattern = "compliance_exceptions/*.txt" + +# Include required specifications here +[[specification]] +source = "../specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "../specification/s3-encryption/data-format/metadata-strategy.md" +[[specification]] +source = "../specification/s3-encryption/client.md" +[[specification]] +source = "../specification/s3-encryption/decryption.md" +[[specification]] +source = "../specification/s3-encryption/encryption.md" +[[specification]] +source = "../specification/s3-encryption/key-derivation.md" +[[specification]] +source = "../specification/s3-encryption/key-commitment.md" + +[report.html] +enabled = true + +# Enable snapshots to prevent requirement coverage regressions +[report.snapshot] +enabled = false diff --git a/test-server/php-v3-server/.gitignore b/test-server/php-v3-server/.gitignore new file mode 100644 index 00000000..07108589 --- /dev/null +++ b/test-server/php-v3-server/.gitignore @@ -0,0 +1,4 @@ +vendor/* +cookies.txt +server.pid +composer.lock \ No newline at end of file diff --git a/test-server/php-v3-server/Makefile b/test-server/php-v3-server/Makefile new file mode 100644 index 00000000..9460d4ed --- /dev/null +++ b/test-server/php-v3-server/Makefile @@ -0,0 +1,39 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: build-server start-server stop-server wait-for-server + +PID_FILE := server.pid +PORT := 8093 + +build-server: + @echo "Building PHP V3 server..." + composer install + +start-server: + @echo "Starting PHP V3 server..." + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + composer run start --timeout=0 > server.log 2>&1 & echo $$! > $(PID_FILE) + @echo "PHP V3 server starting..." + +stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true + @if [ -f $(PID_FILE) ]; then \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ + fi + @rm -f server.log + @echo "Server stopped" + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT) + +duvet: + duvet report + +view-report-mac: + open .duvet/reports/report.html diff --git a/test-server/php-v3-server/README.md b/test-server/php-v3-server/README.md new file mode 100644 index 00000000..284c6e97 --- /dev/null +++ b/test-server/php-v3-server/README.md @@ -0,0 +1,66 @@ +# S3EC PHP v3 Test Server + +This is the PHP V3 implementation of the S3ECTestServer framework. It provides a server implementation for testing S3 Encryption Client functionality. + +## Overview + +The S3ECPhpV3TestServer implements the S3ECTestServer service defined in the shared Smithy model. It provides endpoints for: + +- Creating S3 Encryption Clients with session-based caching +- Putting objects with encryption +- Getting and decrypting objects + +## Starting the Server + +### Method 1: Using Composer (Recommended) +```bash +composer run start +``` + +The server will start on port `8093`. + +## Available Endpoints + +### Server Status +- **GET /** - Returns server status and available endpoints + +### Client Management +- **POST /client** - Creates an S3EncryptionClient and caches it with session persistence +- **GET /cache** - Shows current session state and cached clients (for debugging) + +### Object Operations +- **GET /object/{bucket}/{key}** - Handle GET requests using the S3EncryptionClient +- **PUT /object/{bucket}/{key}** - Handle PUT requests using the S3EncryptionClient + +## Testing with curl + +### Important: Session Cookie Management + +To properly test the server and maintain session persistence, you **must** use cookies with curl: + +#### First Request (creates session cookie): +```bash +curl -X POST http://localhost:8093/client \ + -H "Content-Type: application/json" \ + -v +``` + +#### Subsequent Requests (reuses session cookie): +```bash +curl -X POST http://localhost:8093/client \ + -H "Content-Type: application/json" \ + -v +``` + +#### Check Cache Status: +```bash +curl http://localhost:8093/cache \ + -b cookies.txt +``` + +#### Helpful Notes +- **Session Storage**: Client configurations are stored in `$_SESSION['s3ecCache']` +- **Object Recreation**: AWS SDK objects are recreated from stored configuration (they cannot be serialized) +AWS SDK obbjects cannot be serialized due to internal resources and closures. +- **Helper Function**: `getCachedClient($clientId)` retrieves and recreates clients from cache +- **Debugging**: Enhanced logging and `/cache` endpoint for troubleshooting diff --git a/test-server/php-v3-server/compliance_exceptions/client.txt b/test-server/php-v3-server/compliance_exceptions/client.txt new file mode 100644 index 00000000..0efb20bd --- /dev/null +++ b/test-server/php-v3-server/compliance_exceptions/client.txt @@ -0,0 +1,170 @@ +// +// The PHP V3 implementation is missing the following features: +// +// 1. Client Configuration Options: +// - Legacy algorithm support controls (wrapping algorithms, unauthenticated modes) +// - Uses V3/V3_AND_LEGACY instead +// - Delayed authentication mode configuration +// - Buffer size configuration for memory management +// - Raw keyring material (RSA, AES) +// - SDK client configuration inheritance (credentials, KMS client config) +// - Custom randomness source configuration +// +// 2. Api Operations: +// - DeleteObject and DeleteObjects (with instruction file cleanup) +// - Multipart upload operations (UploadPart, CompleteMultipartUpload, AbortMultipartUpload) +// - ReEncryptInstructionFile for key rotation +// - Non-encryption related S3 operations + +//= ../specification/s3-encryption/client.md#aws-sdk-compatibility +//= type=exception +//# The S3EC SHOULD support invoking operations unrelated to client-side encryption e.g. + +//= ../specification/s3-encryption/client.md#cryptographic-materials +//= type=exception +//# If both a CMM and a Keyring are provided, the S3EC MUST throw an exception. + +//= ../specification/s3-encryption/client.md#cryptographic-materials +//= type=exception +//# When a Keyring is provided, the S3EC MUST create an instance of the DefaultCMM using the provided Keyring. + +//= ../specification/s3-encryption/client.md#enable-legacy-wrapping-algorithms +//= type=exception +//# The option to enable legacy wrapping algorithms MUST be set to false by default. + +//= ../specification/s3-encryption/client.md#enable-legacy-unauthenticated-modes +//= type=exception +//# The S3EC MUST support the option to enable or disable legacy unauthenticated modes (content encryption algorithms). + +//= ../specification/s3-encryption/client.md#enable-legacy-unauthenticated-modes +//= type=exception +//# The option to enable legacy unauthenticated modes MUST be set to false by default. + +//= ../specification/s3-encryption/client.md#enable-legacy-unauthenticated-modes +//= type=exception +//# When enabled, the S3EC MUST be able to decrypt objects encrypted with all content encryption algorithms (both legacy and fully supported). + +//= ../specification/s3-encryption/client.md#enable-legacy-unauthenticated-modes +//= type=exception +//# When disabled, the S3EC MUST NOT decrypt objects encrypted using legacy content encryption algorithms; +//# it MUST throw an exception when attempting to decrypt an object encrypted with a legacy content encryption algorithm. + +//= ../specification/s3-encryption/client.md#enable-delayed-authentication +//= type=exception +//# The S3EC MUST support the option to enable or disable Delayed Authentication mode. + +//= ../specification/s3-encryption/client.md#enable-delayed-authentication +//= type=exception +//# Delayed Authentication mode MUST be set to false by default. + +//= ../specification/s3-encryption/client.md#enable-delayed-authentication +//= type=exception +//# When enabled, the S3EC MAY release plaintext from a stream which has not been authenticated. + +//= ../specification/s3-encryption/client.md#enable-delayed-authentication +//= type=exception +//# When disabled the S3EC MUST NOT release plaintext from a stream which has not been authenticated. + +//= ../specification/s3-encryption/client.md#set-buffer-size +//= type=exception +//# The S3EC SHOULD accept a configurable buffer size which refers to the maximum ciphertext length in bytes to store in memory when Delayed Authentication mode is disabled. + +//= ../specification/s3-encryption/client.md#set-buffer-size +//= type=exception +//# If Delayed Authentication mode is enabled, and the buffer size has been set to a value other than its default, the S3EC MUST throw an exception. + +//= ../specification/s3-encryption/client.md#set-buffer-size +//= type=exception +//# If Delayed Authentication mode is disabled, and no buffer size is provided, the S3EC MUST set the buffer size to a reasonable default. + +//= ../specification/s3-encryption/client.md#cryptographic-materials +//= type=exception +//# The S3EC MAY accept key material directly. + +//= ../specification/s3-encryption/client.md#inherited-sdk-configuration +//= type=exception +//# The S3EC MAY support directly configuring the wrapped SDK clients through its initialization. + +//= ../specification/s3-encryption/client.md#inherited-sdk-configuration +//= type=exception +//# For example, the S3EC MAY accept a credentials provider instance during its initialization. + +//= ../specification/s3-encryption/client.md#inherited-sdk-configuration +//= type=exception +//# If the S3EC accepts SDK client configuration, the configuration MUST be applied to all wrapped S3 clients. + +//= ../specification/s3-encryption/client.md#inherited-sdk-configuration +//= type=exception +//# If the S3EC accepts SDK client configuration, the configuration MUST be applied to all wrapped SDK clients including the KMS client. + +//= ../specification/s3-encryption/client.md#randomness +//= type=exception +//# The S3EC MAY accept a source of randomness during client initialization. + +//= ../specification/s3-encryption/client.md#required-api-operations +//= type=exception +//# - DeleteObject MUST be implemented by the S3EC. + +//= ../specification/s3-encryption/client.md#required-api-operations +//= type=exception +//# - DeleteObject MUST delete the given object key. + +//= ../specification/s3-encryption/client.md#required-api-operations +//= type=exception +//# - DeleteObject MUST delete the associated instruction file using the default instruction file suffix. + +//= ../specification/s3-encryption/client.md#required-api-operations +//= type=exception +//# - DeleteObjects MUST be implemented by the S3EC. + +//= ../specification/s3-encryption/client.md#required-api-operations +//= type=exception +//# - DeleteObjects MUST delete each of the given objects. + +//= ../specification/s3-encryption/client.md#required-api-operations +//= type=exception +//# - DeleteObjects MUST delete each of the corresponding instruction files using the default instruction file suffix. + +//= ../specification/s3-encryption/client.md#optional-api-operations +//= type=exception +//# - UploadPart MAY be implemented by the S3EC. + +//= ../specification/s3-encryption/client.md#optional-api-operations +//= type=exception +//# - UploadPart MUST encrypt each part. + +//= ../specification/s3-encryption/client.md#optional-api-operations +//= type=exception +//# - Each part MUST be encrypted in sequence. + +//= ../specification/s3-encryption/client.md#optional-api-operations +//= type=exception +//# - Each part MUST be encrypted using the same cipher instance for each part. + +//= ../specification/s3-encryption/client.md#optional-api-operations +//= type=exception +//# - CompleteMultipartUpload MAY be implemented by the S3EC. + +//= ../specification/s3-encryption/client.md#optional-api-operations +//= type=exception +//# - CompleteMultipartUpload MUST complete the multipart upload. + +//= ../specification/s3-encryption/client.md#optional-api-operations +//= type=exception +//# - AbortMultipartUpload MAY be implemented by the S3EC. + +//= ../specification/s3-encryption/client.md#optional-api-operations +//= type=exception +//# - AbortMultipartUpload MUST abort the multipart upload. + +//= ../specification/s3-encryption/client.md#optional-api-operations +//= type=exception +//# - ReEncryptInstructionFile MAY be implemented by the S3EC. + +//= ../specification/s3-encryption/client.md#optional-api-operations +//= type=exception +//# - ReEncryptInstructionFile MUST decrypt the instruction file's encrypted data key for the given object using the client's CMM. + +//= ../specification/s3-encryption/client.md#optional-api-operations +//= type=exception +//# - ReEncryptInstructionFile MUST re-encrypt the plaintext data key with a provided keyring. diff --git a/test-server/php-v3-server/compliance_exceptions/content-metadata-strategy.txt b/test-server/php-v3-server/compliance_exceptions/content-metadata-strategy.txt new file mode 100644 index 00000000..bb86da72 --- /dev/null +++ b/test-server/php-v3-server/compliance_exceptions/content-metadata-strategy.txt @@ -0,0 +1,34 @@ +// +// The PHP V3 implementation is missing the following features: +// +// 1. METADATA ENCODING: +// - S3 Server "double encoding" support for proper metadata decoding +// +// 2. INSTRUCTION FILE OPERATIONS: +// - Re-encryption/key rotation via instruction files +// - Custom instruction file suffix support for GetObject requests +// + +//= ../specification/s3-encryption/data-format/metadata-strategy.md#object-metadata +//= type=exception +//# The S3EC SHOULD support decoding the S3 Server's "double encoding". + +//= ../specification/s3-encryption/data-format/metadata-strategy.md#object-metadata +//= type=exception +//# If the S3EC does not support decoding the S3 Server's "double encoding" then it MUST return the content metadata untouched. + +//= ../specification/s3-encryption/data-format/metadata-strategy.md#instruction-file +//= type=exception +//# The S3EC MAY support re-encryption/key rotation via Instruction Files. + +//= ../specification/s3-encryption/data-format/metadata-strategy.md#instruction-file +//= type=exception +//# The S3EC MUST NOT support providing a custom Instruction File suffix on ordinary writes; custom suffixes MUST only be used during re-encryption. + +//= ../specification/s3-encryption/data-format/metadata-strategy.md#instruction-file +//= type=exception +//# The S3EC SHOULD support providing a custom Instruction File suffix on GetObject requests, regardless of whether or not re-encryption is supported. + +//= ../specification/s3-encryption/data-format/metadata-strategy.md#v3-instruction-files +//= type=exception +//# - The V3 message format MUST store the mapkey "x-amz-m" and its value (when present in the content metadata) in the Instruction File. diff --git a/test-server/php-v3-server/compliance_exceptions/content-metadata.txt b/test-server/php-v3-server/compliance_exceptions/content-metadata.txt new file mode 100644 index 00000000..6053a0a6 --- /dev/null +++ b/test-server/php-v3-server/compliance_exceptions/content-metadata.txt @@ -0,0 +1,50 @@ +// +// The PHP V3 implementation is missing the following features: +// +// - Instruction file fallback when object doesn't match V1/V2/V3 formats +// - S3 Server "double encoding" scheme support +// - Writing raw keyring formats (RSA, AES) + +//= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys +//= type=exception +//# - The mapkey "x-amz-key" MUST be present for V1 format objects. + +//= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys +//= type=exception +//# - The mapkey "x-amz-m" SHOULD be present for V3 format objects that use Raw Keyring Material Description. + +//= ../specification/s3-encryption/data-format/content-metadata.md#v3-only +//= type=exception +//# This material description string MAY be encoded by the esoteric double-encoding scheme used by the S3 web server. + +//= ../specification/s3-encryption/data-format/content-metadata.md#v3-only +//= type=exception +//# This encryption context string MAY be encoded by the esoteric double-encoding scheme used by the S3 web server. + +//= ../specification/s3-encryption/data-format/content-metadata.md#v3-only +//= type=exception +//# - The wrapping algorithm value "02" MUST be translated to AES/GCM upon retrieval, and vice versa on write. + +//= ../specification/s3-encryption/data-format/content-metadata.md#v3-only +//= type=exception +//# - The wrapping algorithm value "22" MUST be translated to RSA-OAEP-SHA1 upon retrieval, and vice versa on write. + +//= ../specification/s3-encryption/data-format/content-metadata.md#v1-v2-shared +//= type=exception +//# This string MAY be encoded by the esoteric double-encoding scheme used by the S3 web server. + +//= ../specification/s3-encryption/data-format/content-metadata.md#determining-s3ec-object-status +//= type=exception +//# - If the metadata contains "x-amz-iv" and "x-amz-key" then the object MUST be considered as an S3EC-encrypted object using the V1 format. + +//= ../specification/s3-encryption/data-format/content-metadata.md#determining-s3ec-object-status +//= type=exception +//# If the object matches none of the V1/V2/V3 formats, the S3EC MUST attempt to get the instruction file. + +//= ../specification/s3-encryption/data-format/content-metadata.md#v3-only +//= type=exception +//# The Material Description MUST be used for wrapping algorithms `AES/GCM` (`02`) and `RSA-OAEP-SHA1` (`22`). + +//= ../specification/s3-encryption/data-format/content-metadata.md#v3-only +//= type=exception +//# If the mapkey is not present, the default Material Description value MUST be set to an empty map (`{}`). diff --git a/test-server/php-v3-server/compliance_exceptions/decryption.txt b/test-server/php-v3-server/compliance_exceptions/decryption.txt new file mode 100644 index 00000000..df86d896 --- /dev/null +++ b/test-server/php-v3-server/compliance_exceptions/decryption.txt @@ -0,0 +1,25 @@ +// +// The PHP V3 implementation is missing the following features: +// +// - Support for "range" parameter on GetObject for partial downloads and decryption +// + +//= ../specification/s3-encryption/decryption.md#ranged-gets +//= type=exception +//# The S3EC MAY support the "range" parameter on GetObject which specifies a subset of bytes to download and decrypt. + +//= ../specification/s3-encryption/decryption.md#ranged-gets +//= type=exception +//# If the S3EC supports Ranged Gets, the S3EC MUST adjust the customer-provided range to include the beginning and end of the cipher blocks for the given range. + +//= ../specification/s3-encryption/decryption.md#ranged-gets +//= type=exception +//# If the object was encrypted with ALG_AES_256_GCM_IV12_TAG16_NO_KDF, then ALG_AES_256_CTR_IV16_TAG16_NO_KDF MUST be used to decrypt the range of the object. + +//= ../specification/s3-encryption/decryption.md#ranged-gets +//= type=exception +//# If the object was encrypted with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, then ALG_AES_256_CTR_HKDF_SHA512_COMMIT_KEY MUST be used to decrypt the range of the object. + +//= ../specification/s3-encryption/decryption.md#ranged-gets +//= type=exception +//# If the GetObject response contains a range, but the GetObject request does not contain a range, the S3EC MUST throw an exception. diff --git a/test-server/php-v3-server/compliance_exceptions/encryption.txt b/test-server/php-v3-server/compliance_exceptions/encryption.txt new file mode 100644 index 00000000..5ae44c91 --- /dev/null +++ b/test-server/php-v3-server/compliance_exceptions/encryption.txt @@ -0,0 +1,26 @@ +// +// The PHP V3 implementation is missing the following features: +// +// - Support for "range" parameter on GetObject for partial downloads and decryption +// +// The PHP V3 implementation has an extra "feature". +// NOTE that using this feature will cause the message to be unable to be decrypted by other language implementations. + +// - Support for AAD during content encryption +// + +//= ../specification/s3-encryption/encryption.md#alg-aes-256-ctr-iv16-tag16-no-kdf +//= type=exception +//# Attempts to encrypt using AES-CTR MUST fail. + +//= ../specification/s3-encryption/encryption.md#alg-aes-256-ctr-hkdf-sha512-commit-key +//= type=exception +//# Attempts to encrypt using key committing AES-CTR MUST fail. + +//= ../specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf +//= type=exception +//# The client MUST NOT provide any AAD when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + +//= ../specification/s3-encryption/encryption.md#cipher-initialization +//= type=exception +//# The client SHOULD validate that the generated IV or Message ID is not zeros. diff --git a/test-server/php-v3-server/composer.json b/test-server/php-v3-server/composer.json new file mode 100644 index 00000000..32c2b00c --- /dev/null +++ b/test-server/php-v3-server/composer.json @@ -0,0 +1,36 @@ +{ + "name": "aws/s3ec-php-v3-test-server", + "description": "PHP v3 implementation of the S3EC Test Server framework", + "type": "project", + "license": "Apache-2.0", + "repositories": [ + { + "type": "path", + "url": "./local-php-sdk", + "options": { + "symlink": true + } + } + ], + "require": { + "php": ">=7.4", + "aws/aws-sdk-php": "@dev", + "ramsey/uuid": "^4.9" + }, + "autoload": { + "psr-4": { + "S3EC\\PhpV2Server\\": "src/" + } + }, + "scripts": { + "start": [ + "php -S 0.0.0.0:8093 src/index.php" + ] + }, + "config": { + "optimize-autoloader": true, + "platform": { + "php": "8.1" + } + } +} \ No newline at end of file diff --git a/test-server/php-v3-server/local-php-sdk b/test-server/php-v3-server/local-php-sdk new file mode 160000 index 00000000..f53d8fc6 --- /dev/null +++ b/test-server/php-v3-server/local-php-sdk @@ -0,0 +1 @@ +Subproject commit f53d8fc6cdbc1e64e7d14e72d1e315d05003b2b4 diff --git a/test-server/php-v3-server/src/client.php b/test-server/php-v3-server/src/client.php new file mode 100644 index 00000000..f57c643a --- /dev/null +++ b/test-server/php-v3-server/src/client.php @@ -0,0 +1,77 @@ +toString(); + $kmsKeyId = $keyMaterial["kmsKeyId"] ?? null; + $commitmentPolicy = $configData['commitmentPolicy'] ?? "REQUIRE_ENCRYPT_REQUIRE_DECRYPT"; + $instFileConfig = $configData['instructionFileConfig'] ?? null; + $instFilePut = false; + if ($instFileConfig != null) { + $instFilePut = $instFileConfig['enableInstructionFilePutObject'] ?? false; + } + + + if (empty($configData)) { + return GenericServerError("Invalid config in request body", 400); + } + if (is_null($keyMaterial) || is_null($kmsKeyId)) { + return GenericServerError("Invalid keyMaterial in config", 400); + } + + // Store client configuration instead of objects (AWS objects can't be serialized) + $_SESSION['s3ecCache'][$clientId] = [ + 's3Config' => [ + 'region' => 'us-west-2', + 'version' => 'latest', + 'http' => [ + 'debug' => false, + 'verify' => true, + 'curl' => [ + CURLOPT_VERBOSE => false, + CURLOPT_NOPROGRESS => true + ] + ] + ], + 'kmsConfig' => [ + 'region' => 'us-west-2', + 'version' => 'latest', + 'http' => [ + 'debug' => false, + 'verify' => true, + 'curl' => [ + CURLOPT_VERBOSE => false, + CURLOPT_NOPROGRESS => true + ] + ] + ], + 'kmsKeyId' => $kmsKeyId, + 'legacy' => $legacyAlgorithms, + 'commitmentPolicy' => $commitmentPolicy, + 'instFilePut' => $instFilePut, + 'created' => time() + ]; + + // Auto-update cookies.txt with current session ID so tests can access cached clients + writeSessionIdToCookiesFile(session_id()); + + header("Content-Type: application/json"); + return json_encode([ + 'clientId' => $clientId, + ]); +} diff --git a/test-server/php-v3-server/src/errors.php b/test-server/php-v3-server/src/errors.php new file mode 100644 index 00000000..2b59861d --- /dev/null +++ b/test-server/php-v3-server/src/errors.php @@ -0,0 +1,42 @@ + 'GenericServerError', + 'message' => $message + ]; + + return json_encode($errorResponse); +} + +/** + * Used for modeled errors, e.g. errors thrown by the S3EC + * Tests SHOULD expect this error in negative tests. + * + * @param string $message The error message to include in the response + * @return string JSON-encoded error response + */ +function S3EncryptionClientError($message) +{ + http_response_code(500); + header('Content-Type: application/json'); + + $errorResponse = [ + "__type" => "software.amazon.encryption.s3#S3EncryptionClientError", + 'message' => $message + ]; + + return json_encode($errorResponse); +} diff --git a/test-server/php-v3-server/src/get_object.php b/test-server/php-v3-server/src/get_object.php new file mode 100644 index 00000000..fbd42f7a --- /dev/null +++ b/test-server/php-v3-server/src/get_object.php @@ -0,0 +1,108 @@ + $legacy, + '@MaterialsProvider' => $materialProvider, + '@KmsEncryptionContext' => $encryptionContext, + '@CommitmentPolicy' => $commitmentPolicy, + 'Bucket' => $bucket, + 'Key' => $key, + ]; + + // Add custom instruction file suffix if provided + if (!is_null($instructionFileSuffix) && !empty($instructionFileSuffix)) { + $getObjectParams['@InstructionFileSuffix'] = $instructionFileSuffix; + } + + $result = $s3ec->getObject($getObjectParams); + + // Capture and discard any unwanted output from AWS SDK + $unwantedOutput = ob_get_clean(); + if (!empty($unwantedOutput)) { + error_log("AWS SDK produced unexpected output: " . strlen($unwantedOutput) . " bytes"); + } + + $body = $result['Body']->getContents(); + $formattedMetadata = formatMetadataForResponse($result["Metadata"]); + + // Now set headers safely + header("Content-Metadata: " . $formattedMetadata); + header("Content-Type: application/octet-stream"); + header("Content-Length: " . strlen($body)); + return $body; + } catch (InvalidArgumentException $e) { + // Clean up output buffer if still active + if (ob_get_level()) { + ob_end_clean(); + } + return GenericServerError("Invalid argument: " . $e->getMessage(), 400); + } catch (Exception $e) { + // Clean up output buffer if still active + if (ob_get_level()) { + ob_end_clean(); + } + if (strpos($e->getMessage(), "@SecurityProfile=V3") !== false) { + return S3EncryptionClientError($e->getMessage()); + } elseif (strpos($e->getMessage(), "Provided encryption context does not match information retrieved from S3") !== false) { + return S3EncryptionClientError($e->getMessage()); + } elseif (strpos($e->getMessage(), "Message is encrypted with a non commiting algorithm but commitment policy is set to REQUIRE_ENCRYPT_REQUIRE_DECRYPT. Select a valid commitment policy to decrypt this object.") !== false) { + return S3EncryptionClientError($e->getMessage()); + } elseif (strpos($e->getMessage(), "One or more reserved keys found in Instruction file when they should not be present.") !== false) { + return S3EncryptionClientError($e->getMessage()); + } elseif (strpos($e->getMessage(), "Expected a V3 envelope but was unable to constuct one.") !== false) { + return S3EncryptionClientError($e->getMessage()); + } elseif (strpos($e->getMessage(), "Malformed metadata envelope.") !== false) { + return S3EncryptionClientError($e->getMessage()); + } else { + error_log("This is the error: " . $e->getMessage()); + return GenericServerError("Server argument: " . $e->getMessage(), 500); + } + } +} diff --git a/test-server/php-v3-server/src/index.php b/test-server/php-v3-server/src/index.php new file mode 100644 index 00000000..f5f5cdb5 --- /dev/null +++ b/test-server/php-v3-server/src/index.php @@ -0,0 +1,295 @@ += 7 && $parts[5] === 'PHPSESSID') { + error_log("Found session ID in cookies.txt: " . $parts[6]); + return $parts[6]; // Return the session ID value + } + } + + error_log("No PHPSESSID found in cookies.txt file"); + return null; +} + +// Function to write session ID to cookies.txt file +function writeSessionIdToCookiesFile($sessionId) +{ + $cookiesFile = __DIR__ . '/../cookies.txt'; + + // Create Netscape cookie format entry + $cookieLine = "localhost\tFALSE\t/\tFALSE\t0\tPHPSESSID\t$sessionId"; + + // Write header and cookie entry + $content = "# Netscape HTTP Cookie File\n"; + $content .= "# https://curl.se/docs/http-cookies.html\n"; + $content .= "# This file was generated by libcurl! Edit at your own risk.\n\n"; + $content .= $cookieLine . "\n"; + + $result = file_put_contents($cookiesFile, $content); + + if ($result === false) { + error_log("Failed to write session ID to cookies.txt file: $cookiesFile"); + return false; + } + + error_log("Successfully wrote session ID to cookies.txt: $sessionId"); + return true; +} + +set_time_limit(600); +// Start session to persist cache across requests +// First try to use session ID from cookies.txt if available +$sessionId = getSessionIdFromCookiesFile(); +if ($sessionId) { + session_id($sessionId); +} +session_start(); + +// Initialize session cache if it doesn't exist +if (!isset($_SESSION['s3ecCache'])) { + $_SESSION['s3ecCache'] = []; +} + +// Simple router class +class SimpleRouter +{ + private $routes = []; + + public function addRoute($method, $path, $handler) + { + $this->routes[] = [ + 'method' => strtoupper($method), + 'path' => $path, + 'handler' => $handler + ]; + } + + public function handleRequest() + { + $method = $_SERVER['REQUEST_METHOD']; + $path = parse_url($_SERVER['REQUEST_URI'], PHP_URL_PATH); + + foreach ($this->routes as $route) { + if ($route['method'] === $method) { + $params = $this->matchPathWithParams($route['path'], $path); + if ($params !== false) { + return call_user_func($route['handler'], $params); + } + } + } + + // Default 404 response + http_response_code(404); + return json_encode(['error' => 'Not Found']); + } + + private function matchPathWithParams($routePath, $requestPath) + { + // Handle exact matches first (for routes without parameters) + if ($routePath === $requestPath) { + return []; + } + + // Convert route path like '/object/{bucket}/{key}' to regex + $pattern = preg_replace('/\{([^}]+)\}/', '([^/]+)', $routePath); + $pattern = '/^' . str_replace('/', '\/', $pattern) . '$/'; + + if (preg_match($pattern, $requestPath, $matches)) { + array_shift($matches); // Remove full match + + // Extract parameter names + preg_match_all('/\{([^}]+)\}/', $routePath, $paramNames); + $params = []; + + foreach ($paramNames[1] as $index => $paramName) { + $params[$paramName] = $matches[$index] ?? null; + } + + return $params; + } + + return false; + } +} + +// Helper function to get cached client by ID +function getCachedClient($clientId) +{ + if (!isset($_SESSION['s3ecCache'][$clientId])) { + return null; + } + + $config = $_SESSION['s3ecCache'][$clientId]; + + // Recreate the AWS clients from stored configuration + $s3Client = new S3Client($config['s3Config']); + $encryptionClient = new S3EncryptionClientV3($s3Client); + + $kmsClient = new KmsClient($config['kmsConfig']); + $materialsProvider = new KmsMaterialsProviderV3($kmsClient, $config['kmsKeyId']); + + return [ + 's3Client' => $s3Client, + 'encryptionClient' => $encryptionClient, + 'materialsProvider' => $materialsProvider, + 'config' => $config + ]; +} + +function createDefaultClientTuple(): array +{ + $s3Client = new S3Client([ + 'region' => 'us-west-2', + 'version' => 'latest', + 'http' => [ + 'debug' => false, + 'verify' => true, + 'curl' => [ + CURLOPT_VERBOSE => false, + CURLOPT_NOPROGRESS => true + ] + ] + ]); + $encryptionClient = new S3EncryptionClientV3($s3Client); + + $kmsClient = new KmsClient([ + 'region' => 'us-west-2', + 'version' => 'latest', + 'http' => [ + 'debug' => false, + 'verify' => true, + 'curl' => [ + CURLOPT_VERBOSE => false, + CURLOPT_NOPROGRESS => true + ] + ] + ]); + $materialsProvider = new KmsMaterialsProviderV3($kmsClient, 'arn:aws:kms:us-west-2:370957321024:alias/S3EC-Test-Server-Github-KMS-Key'); + + return [ + 'encryptionClient' => $encryptionClient, + 'materialsProvider' => $materialsProvider + ]; +} + +function metadataStringToMap($metadata): array +{ + $md = []; + + if (empty($metadata)) { + return $md; + } + + $mdList = explode(',', $metadata); + + foreach ($mdList as $entry) { + $parts = explode(']:[', $entry); + + if (count($parts) === 2) { + $key = substr($parts[0], 1); + $value = substr($parts[1], 0, -1); + $md[$key] = $value; + } else { + throw new InvalidArgumentException("Malformed metadata list entry: " . $entry); + } + } + + return $md; +} +function formatMetadataForResponse($metadata) +{ + $metadataList = []; + // Handle different metadata input types + if (is_array($metadata)) { + // If it's an associative array (like Python dict) + foreach ($metadata as $key => $value) { + $metadataList[] = $key . '=' . $value; + } + } elseif (is_string($metadata) && !empty($metadata)) { + // If it's already a string, assume it's in the correct format + return $metadata; + } + + // Convert array to comma-separated string + return implode(',', $metadataList); +} + +// Initialize router +$router = new SimpleRouter(); + +// Add basic routes +$router->addRoute('GET', '/', function () { + return json_encode([ + 'service' => 'S3EC PHP v2 Test Server', + 'status' => 'running', + 'port' => 8087, + 'endpoints' => [ + 'GET /' => 'Server status', + 'POST /client' => 'Create an S3EncryptionClient and cache it.', + 'GET /object/{bucket}/{key}' => 'Handle GET requests to /object/{bucket}/{key} by using the S3EncryptionClient to make a GetObject request to S3.', + 'PUT /object/{bucket}/{key}' => 'Handle PUT requests to /object/{bucket}/{key} by using the S3EncryptionClient to make a PutObject request to S3.', + ] + ]); +}); + +$router->addRoute('GET', '/cache', function () { + return json_encode([ + 'sessionId' => session_id(), + 'sessionStatus' => session_status(), + 'totalCachedClients' => count($_SESSION['s3ecCache'] ?? []), + 'allClientIds' => array_keys($_SESSION['s3ecCache'] ?? []), + 'cacheDetails' => $_SESSION['s3ecCache'] ?? [] + ]); +}); + +$router->addRoute('GET', '/object/{bucket}/{key}', function ($params) { + return handleGetObject($params); +}); + +$router->addRoute('PUT', '/object/{bucket}/{key}', function ($params) { + return handlePutObject($params); +}); + +$router->addRoute('POST', '/client', function () { + return handleCreateClient(); +}); + +// Handle the request and output response +$result = $router->handleRequest(); +if ($result !== false) { + echo $result; +} diff --git a/test-server/php-v3-server/src/put_object.php b/test-server/php-v3-server/src/put_object.php new file mode 100644 index 00000000..2f882b1e --- /dev/null +++ b/test-server/php-v3-server/src/put_object.php @@ -0,0 +1,82 @@ + 'gcm', + 'KeySize' => 256, + ]; + $legacyConfig = $s3ecClientTuple["legacy"] ?? false; + $legacy = null; + if ($legacyConfig === false) { + $legacy = "V2"; + } else { + $legacy = "V2_AND_LEGACY"; + } + $commitmentPolicy = $s3ecClientTuple['config']['commitmentPolicy']; + $strategy = $s3ecClientTuple["config"]["instFilePut"] ? + new InstructionFileMetadataStrategy($s3Client) : + new HeadersMetadataStrategy(); + + try { + $result = $s3ec->putObject([ + '@SecurityProfile' => $legacy, + '@MaterialsProvider' => $materialProvider, + '@CommitmentPolicy' => $commitmentPolicy, + '@KmsEncryptionContext' => $encryptionContext, + '@MetadataStrategy' => $strategy, + '@CipherOptions' => $cipherOptions, + 'Bucket' => $bucket, + 'Key' => $key, + 'Body' => $rawBody, + ]); + + header("Content-Type: application/json"); + return json_encode([ + "bucket" => $bucket, + "key" => $key, + // php for some reason blows java's heap if we pass the metadata + // "metadata" => $encryptionContext + ]); + + } catch (InvalidArgumentException $e) { + return S3EncryptionClientError("Invalid arguement: " . $e->getMessage()); + } catch (Exception $e) { + return GenericServerError("Server error: " . $e->getMessage()); + } +} diff --git a/test-server/python-v3-server/.duvet/.gitignore b/test-server/python-v3-server/.duvet/.gitignore new file mode 100644 index 00000000..93956e36 --- /dev/null +++ b/test-server/python-v3-server/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ \ No newline at end of file diff --git a/test-server/python-v3-server/.duvet/config.toml b/test-server/python-v3-server/.duvet/config.toml new file mode 100644 index 00000000..09dbe6d3 --- /dev/null +++ b/test-server/python-v3-server/.duvet/config.toml @@ -0,0 +1,22 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "**/*.py" +comment-style = { meta = "##=", content = "##%" } + +# Include required specifications here +[[specification]] +source = "../specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "../specification/s3-encryption/data-format/metadata-strategy.md" +[[specification]] +source = "../specification/s3-encryption/encryption.md" +[[specification]] +source = "../specification/s3-encryption/key-derivation.md" + +[report.html] +enabled = true + +# Enable snapshots to prevent requirement coverage regressions +[report.snapshot] +enabled = false diff --git a/test-server/python-server/.gitignore b/test-server/python-v3-server/.gitignore similarity index 100% rename from test-server/python-server/.gitignore rename to test-server/python-v3-server/.gitignore diff --git a/test-server/python-v3-server/Makefile b/test-server/python-v3-server/Makefile new file mode 100644 index 00000000..930c950c --- /dev/null +++ b/test-server/python-v3-server/Makefile @@ -0,0 +1,42 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: build-server start-server stop-server wait-for-server + +PID_FILE := server.pid +PORT := 8081 + +build-server: + @echo "Building Python V3 server..." + python -m venv .venv + .venv/bin/python -m ensurepip + .venv/bin/python -m pip install -e . + .venv/bin/python -m pip install -e ../.. + +start-server: + @echo "Starting Python V3 server..." + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + .venv/bin/python src/main.py > server.log 2>&1 & echo $$! > $(PID_FILE) + @echo "Python V3 server starting..." + +stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true + @if [ -f $(PID_FILE) ]; then \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ + fi + @rm -f server.log + @echo "Server stopped" + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT) + +duvet: + duvet report + +view-report-mac: + open .duvet/reports/report.html diff --git a/test-server/python-server/README.md b/test-server/python-v3-server/README.md similarity index 100% rename from test-server/python-server/README.md rename to test-server/python-v3-server/README.md diff --git a/test-server/python-server/poetry.lock b/test-server/python-v3-server/poetry.lock similarity index 100% rename from test-server/python-server/poetry.lock rename to test-server/python-v3-server/poetry.lock diff --git a/test-server/python-server/pyproject.toml b/test-server/python-v3-server/pyproject.toml similarity index 100% rename from test-server/python-server/pyproject.toml rename to test-server/python-v3-server/pyproject.toml diff --git a/test-server/python-server/src/__init__.py b/test-server/python-v3-server/src/__init__.py similarity index 100% rename from test-server/python-server/src/__init__.py rename to test-server/python-v3-server/src/__init__.py diff --git a/test-server/python-server/src/main.py b/test-server/python-v3-server/src/main.py similarity index 100% rename from test-server/python-server/src/main.py rename to test-server/python-v3-server/src/main.py diff --git a/test-server/python-server/tests/__init__.py b/test-server/python-v3-server/tests/__init__.py similarity index 100% rename from test-server/python-server/tests/__init__.py rename to test-server/python-v3-server/tests/__init__.py diff --git a/test-server/ruby-v2-server/.duvet/.gitignore b/test-server/ruby-v2-server/.duvet/.gitignore new file mode 100644 index 00000000..93956e36 --- /dev/null +++ b/test-server/ruby-v2-server/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ \ No newline at end of file diff --git a/test-server/ruby-v2-server/.duvet/config.toml b/test-server/ruby-v2-server/.duvet/config.toml new file mode 100644 index 00000000..7a34c0ff --- /dev/null +++ b/test-server/ruby-v2-server/.duvet/config.toml @@ -0,0 +1,33 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "local-ruby-sdk/gems/aws-sdk-s3/lib/**/*.rb" +comment-style = { meta = "##=", content = "##%" } + +[[source]] +pattern = "local-ruby-sdk/gems/aws-sdk-s3/spec/**/*.rb" +comment-style = { meta = "##=", content = "##%" } + +# Include required specifications here +[[specification]] +source = "../specification/s3-encryption/client.md" +[[specification]] +source = "../specification/s3-encryption/decryption.md" +[[specification]] +source = "../specification/s3-encryption/encryption.md" +[[specification]] +source = "../specification/s3-encryption/key-commitment.md" +[[specification]] +source = "../specification/s3-encryption/key-derivation.md" +[[specification]] +source = "../specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "../specification/s3-encryption/data-format/metadata-strategy.md" + + +[report.html] +enabled = true + +# Enable snapshots to prevent requirement coverage regressions +[report.snapshot] +enabled = false diff --git a/test-server/ruby-v2-server/.gitignore b/test-server/ruby-v2-server/.gitignore new file mode 100644 index 00000000..d20a29c0 --- /dev/null +++ b/test-server/ruby-v2-server/.gitignore @@ -0,0 +1,2 @@ +vendor +server.pid \ No newline at end of file diff --git a/test-server/ruby-v2-server/Gemfile b/test-server/ruby-v2-server/Gemfile new file mode 100644 index 00000000..2759a87b --- /dev/null +++ b/test-server/ruby-v2-server/Gemfile @@ -0,0 +1,15 @@ +source 'https://rubygems.org' + +ruby '~> 3.0' + +gem 'sinatra', '~> 3.0' +gem 'puma', '~> 6.0' +gem 'aws-sdk-s3', path: 'local-ruby-sdk/gems/aws-sdk-s3' +gem 'aws-sdk-kms', path: 'local-ruby-sdk/gems/aws-sdk-kms' +gem 'json', '~> 2.0' +gem 'concurrent-ruby', '~> 1.0' +gem 'nokogiri', '~> 1.13' + +group :development do + gem 'rubocop', '~> 1.0' +end diff --git a/test-server/ruby-v2-server/Gemfile.lock b/test-server/ruby-v2-server/Gemfile.lock new file mode 100644 index 00000000..b9f08375 --- /dev/null +++ b/test-server/ruby-v2-server/Gemfile.lock @@ -0,0 +1,111 @@ +PATH + remote: local-ruby-sdk/gems/aws-sdk-kms + specs: + aws-sdk-kms (1.118.0) + aws-sdk-core (~> 3, >= 3.239.1) + aws-sigv4 (~> 1.5) + +PATH + remote: local-ruby-sdk/gems/aws-sdk-s3 + specs: + aws-sdk-s3 (1.206.0) + aws-sdk-core (~> 3, >= 3.234.0) + aws-sdk-kms (~> 1) + aws-sigv4 (~> 1.5) + +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.3) + aws-eventstream (1.4.0) + aws-partitions (1.1180.0) + aws-sdk-core (3.239.2) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) + base64 + bigdecimal + jmespath (~> 1, >= 1.6.1) + logger + aws-sigv4 (1.12.1) + aws-eventstream (~> 1, >= 1.0.2) + base64 (0.3.0) + bigdecimal (3.3.1) + bigdecimal (3.3.1-java) + concurrent-ruby (1.3.5) + jmespath (1.6.2) + json (2.13.2) + json (2.13.2-java) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + logger (1.7.0) + mustermann (3.0.4) + ruby2_keywords (~> 0.0.1) + nio4r (2.7.4) + nio4r (2.7.4-java) + nokogiri (1.18.10-arm64-darwin) + racc (~> 1.4) + nokogiri (1.18.10-java) + racc (~> 1.4) + parallel (1.27.0) + parser (3.3.9.0) + ast (~> 2.4.1) + racc + prism (1.5.1) + puma (6.6.1) + nio4r (~> 2.0) + puma (6.6.1-java) + nio4r (~> 2.0) + racc (1.8.1) + racc (1.8.1-java) + rack (2.2.17) + rack-protection (3.2.0) + base64 (>= 0.1.0) + rack (~> 2.2, >= 2.2.4) + rainbow (3.1.1) + regexp_parser (2.11.3) + rubocop (1.80.2) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.46.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.46.0) + parser (>= 3.3.7.2) + prism (~> 1.4) + ruby-progressbar (1.13.0) + ruby2_keywords (0.0.5) + sinatra (3.2.0) + mustermann (~> 3.0) + rack (~> 2.2, >= 2.2.4) + rack-protection (= 3.2.0) + tilt (~> 2.0) + tilt (2.6.1) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.1.0) + +PLATFORMS + arm64-darwin-24 + universal-java-21 + +DEPENDENCIES + aws-sdk-kms! + aws-sdk-s3! + concurrent-ruby (~> 1.0) + json (~> 2.0) + nokogiri (~> 1.13) + puma (~> 6.0) + rubocop (~> 1.0) + sinatra (~> 3.0) + +RUBY VERSION + ruby 3.4.5p51 + +BUNDLED WITH + 2.6.9 diff --git a/test-server/ruby-v2-server/Makefile b/test-server/ruby-v2-server/Makefile new file mode 100644 index 00000000..e0f938fc --- /dev/null +++ b/test-server/ruby-v2-server/Makefile @@ -0,0 +1,43 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: build-server start-server stop-server wait-for-server + +PID_FILE := server.pid +PORT := 8098 + +build-server: + @echo "Building Ruby V2 server..." + bundle install + +start-server: + @if [ -f $(PID_FILE) ]; then \ + echo "❌ Error: Server already running. Stop before starting."; \ + exit 1; \ + fi; + @echo "Starting Ruby V2 server..." + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + PORT=$(PORT) bundle exec ruby app.rb > server.log 2>&1 & echo $$! > $(PID_FILE) + @echo "Ruby V2 server starting..." + +stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true + @if [ -f $(PID_FILE) ]; then \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ + fi + @rm -f server.log + @echo "Server stopped" + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT) + +duvet: + duvet report + +view-report-mac: + open .duvet/reports/report.html diff --git a/test-server/ruby-v2-server/README.md b/test-server/ruby-v2-server/README.md new file mode 100644 index 00000000..4b3e5209 --- /dev/null +++ b/test-server/ruby-v2-server/README.md @@ -0,0 +1,73 @@ +# Ruby S3 Encryption Client Test Server + +This is a Ruby implementation of the S3 Encryption Client test server +that provides an invariant interface around the S3 Encryption Client v2. +It's designed to work alongside other implementations of test servers for cross-language compatibility testing. + +## Overview + +The server provides a REST API that wraps the AWS S3 Encryption Client v2, +allowing tests to verify that all language implementations behave consistently. + +## Endpoints + +- `POST /client` - Create a new S3 encryption client instance +- `PUT /object/{bucket}/{key}` - Encrypt and store an object +- `GET /object/{bucket}/{key}` - Retrieve and decrypt an object +- `GET /health` - Health check endpoint + +## Configuration + +The server runs on port **8086** by default. + +## Setup + +1. Install Ruby 3.x +2. Install dependencies: + + ```bash + cd test-server/ruby-v2-server + bundle install + ``` + +3. Set up AWS credentials (via AWS CLI, environment variables, or IAM roles) + +4. Start the server: + + ```bash + ruby app.rb + # or using Rack + bundle exec rackup -p 8086 + ``` + +## Usage + +The server is designed to be used by the Java test suite in `test-server/java-tests/`. +The tests will automatically discover and use this server for cross-language compatibility testing. + +### Environment Variables + +- `TEST_SERVER_KMS_KEY_ARN` - KMS key ARN for encryption (defaults to test key) +- `TEST_SERVER_S3_BUCKET` - S3 bucket for testing (defaults to test bucket) + +## Architecture + +- `app.rb` - Main Sinatra application +- `lib/client_manager.rb` - Manages S3 encryption client instances +- `lib/metadata_utils.rb` - Handles metadata serialization/deserialization +- `lib/error_handlers.rb` - Smithy-compliant error responses + +## Error Handling + +The server returns errors in the format expected by the Smithy model: + +- `GenericServerError` - Internal server errors +- `S3EncryptionClientError` - Errors from the S3 Encryption Client + +## Compatibility + +This server is compatible with: + +- S3 Encryption Client v2 +- Legacy v1 clients (when `enableLegacyWrappingAlgorithms` is true) +- Cross-language testing with other implementations diff --git a/test-server/ruby-v2-server/app.rb b/test-server/ruby-v2-server/app.rb new file mode 100644 index 00000000..96e55c8a --- /dev/null +++ b/test-server/ruby-v2-server/app.rb @@ -0,0 +1,241 @@ +require 'sinatra' +require 'json' +require_relative 'lib/client_manager' +require_relative 'lib/metadata_utils' +require_relative 'lib/error_handlers' +require_relative 'lib/logger' + +# See: https://github.com/ruby/openssl/issues/949 +Aws.use_bundled_cert! + +class S3ECRubyServer < Sinatra::Base + configure do + set :port, ENV['PORT'] || 8098 + set :bind, '0.0.0.0' + set :show_exceptions, false + set :raise_errors, false + end + + def initialize + super + @client_manager = ClientManager.new + S3ECLogger.info("S3EC_SERVER: Ruby V2 server initialized on port #{settings.port}") + end + + # Request logging middleware + before do + @request_id = S3ECLogger.generate_request_id + S3ECLogger.log_request(request.request_method, request.path_info, request.env, @request_id) + end + + # Response logging middleware + after do + S3ECLogger.log_response(response.status, @request_id) + end + + # Health check endpoint + get '/health' do + content_type :json + { status: 'OK', server: 'Ruby V2 S3EC Test Server', port: settings.port.to_i }.to_json + end + + # POST /client - Create S3 encryption client + post '/client' do + begin + S3ECLogger.debug("CLIENT_ENDPOINT [#{@request_id}]: Processing client creation request") + + # Parse request body + request_body = request.body.read + S3ECLogger.debug("CLIENT_ENDPOINT [#{@request_id}]: Request body size: #{request_body.length} bytes") + + parsed_data = JSON.parse(request_body) + config = parsed_data['config'] || {} + + S3ECLogger.debug("CLIENT_ENDPOINT [#{@request_id}]: Parsed config: #{config.inspect}") + + # Create client using client manager + client_id = @client_manager.create_client(config) + + S3ECLogger.info("CLIENT_ENDPOINT [#{@request_id}]: Successfully created client #{client_id}") + + # Return client ID + content_type :json + { clientId: client_id }.to_json + + rescue JSON::ParserError => e + S3ECLogger.log_error(e, { endpoint: '/client', operation: 'JSON parsing' }, @request_id) + ErrorHandlers.send_generic_server_error(self, "Invalid JSON in request body", 400) + rescue => e + S3ECLogger.log_error(e, { endpoint: '/client', operation: 'client creation', config: config }, @request_id) + ErrorHandlers.send_s3_encryption_client_error(self, "Failed to create client: #{e.message}") + end + end + + # PUT /object/{bucket}/{key} - Encrypt and put object + put '/object/:bucket/:key' do + bucket = params[:bucket] + key = params[:key] + client_id = request.env['HTTP_CLIENTID'] + + begin + S3ECLogger.debug("PUT_ENDPOINT [#{@request_id}]: Processing PUT request for s3://#{bucket}/#{key}") + + # Validate client ID + unless client_id + S3ECLogger.log_validation_error('ClientID', 'missing', @request_id) + ErrorHandlers.send_generic_server_error(self, "ClientID header is required", 400) + end + + # Get client from cache + client = @client_manager.get_client(client_id) + unless client + S3ECLogger.log_validation_error('ClientID', client_id, @request_id) + ErrorHandlers.send_generic_server_error(self, "No client found for ClientID: #{client_id}", 404) + end + + # Get request body + body = request.body.read + S3ECLogger.debug("PUT_ENDPOINT [#{@request_id}]: Request body size: #{body.length} bytes") + + # Parse metadata from header + metadata_header = request.env['HTTP_CONTENT_METADATA'] || '' + encryption_context = MetadataUtils.string_to_map(metadata_header) + S3ECLogger.log_metadata_processing('parse', metadata_header, encryption_context) + + # Prepare S3 put_object parameters + put_params = { + bucket: bucket, + key: key, + body: body + } + + # Add encryption context if present + put_params[:kms_encryption_context] = encryption_context unless encryption_context.empty? + + # Log S3 operation + S3ECLogger.log_s3_operation('put', bucket, key, encryption_context, "ClientID: #{client_id}, BodySize: #{body.length}") + + # Make the put_object request + response = client.put_object(put_params) + + S3ECLogger.info("PUT_ENDPOINT [#{@request_id}]: Successfully put object s3://#{bucket}/#{key}") + + # Prepare response metadata + response_metadata = MetadataUtils.map_to_array(encryption_context) + S3ECLogger.log_metadata_processing('response', encryption_context, response_metadata) + + # Return response matching Smithy model + content_type :json + { + bucket: bucket, + key: key, + metadata: response_metadata + }.to_json + + rescue Aws::S3::EncryptionV2::Errors::EncryptionError, Aws::S3::EncryptionV3::Errors::EncryptionError => e + S3ECLogger.log_error(e, { endpoint: '/put', error_category: 'EncryptionError' }, @request_id) + ErrorHandlers.send_s3_encryption_client_error(self, e.message) + rescue StandardError => e + # Handle generic server errors (return as GenericServerError) + S3ECLogger.log_error(e, { endpoint: '/put', error_category: 'generic_server' }, @request_id) + status_code = e.respond_to?(:code) ? e.code : 500 + ErrorHandlers.send_generic_server_error(self, e.message, status_code) + end + end + + # GET /object/{bucket}/{key} - Get and decrypt object + get '/object/:bucket/:key' do + bucket = params[:bucket] + key = params[:key] + client_id = request.env['HTTP_CLIENTID'] + + begin + S3ECLogger.debug("GET_ENDPOINT [#{@request_id}]: Processing GET request for s3://#{bucket}/#{key}") + + # Validate client ID + unless client_id + S3ECLogger.log_validation_error('ClientID', 'missing', @request_id) + ErrorHandlers.send_generic_server_error(self, "ClientID header is required", 400) + end + + # Get client from cache + client = @client_manager.get_client(client_id) + unless client + S3ECLogger.log_validation_error('ClientID', client_id, @request_id) + ErrorHandlers.send_generic_server_error(self, "No client found for ClientID: #{client_id}", 404) + end + + # Parse metadata from header + metadata_header = request.env['HTTP_CONTENT_METADATA'] || '' + encryption_context = MetadataUtils.string_to_map(metadata_header) + S3ECLogger.log_metadata_processing('parse', metadata_header, encryption_context) + + # Prepare S3 get_object parameters + get_params = { + bucket: bucket, + key: key + } + + # Add custom instruction file suffix if present + instruction_file_suffix = request.env['HTTP_INSTRUCTIONFILESUFFIX'] + if instruction_file_suffix && !instruction_file_suffix.empty? + get_params[:envelope_location] = :instruction_file + get_params[:instruction_file_suffix] = instruction_file_suffix + S3ECLogger.debug("GET_ENDPOINT [#{@request_id}]: Using custom instruction file suffix: #{instruction_file_suffix}") + elsif !encryption_context.empty? + get_params[:kms_encryption_context] = encryption_context + end + + # Log S3 operation + S3ECLogger.log_s3_operation('get', bucket, key, encryption_context, "ClientID: #{client_id}") + + # Make the get_object request + response = client.get_object(get_params) + + # Extract body and metadata + body = response.body.read + metadata = response.metadata || {} + + S3ECLogger.info("GET_ENDPOINT [#{@request_id}]: Successfully got object s3://#{bucket}/#{key}, BodySize: #{body.length}") + + # Set Content-Metadata header in response + metadata_str = MetadataUtils.map_to_string(metadata) + S3ECLogger.log_metadata_processing('response', metadata, metadata_str) + + headers['Content-Metadata'] = metadata_str unless metadata_str.empty? + + # Return the body as response + content_type 'application/octet-stream' + body + + rescue Aws::S3::EncryptionV2::Errors::DecryptionError, Aws::S3::EncryptionV3::Errors::DecryptionError => e + S3ECLogger.log_error(e, { endpoint: '/get', error_category: 'DecryptionError' }, @request_id) + ErrorHandlers.send_s3_encryption_client_error(self, e.message) + rescue StandardError => e + # Handle generic server errors (return as GenericServerError) + S3ECLogger.log_error(e, { endpoint: '/get', error_category: 'generic_server' }, @request_id) + status_code = e.respond_to?(:code) ? e.code : 500 + ErrorHandlers.send_generic_server_error(self, e.message, status_code) + end + end + + # Global error handler + error do + error = env['sinatra.error'] + context = { + endpoint: request.path_info, + method: request.request_method, + params: params, + error_type: 'global_error_handler' + } + + S3ECLogger.log_error(error, context, @request_id) + ErrorHandlers.send_generic_server_error(self, "Internal server error: #{error.message}") + end + + # Start server when run directly + if __FILE__ == $0 + S3ECLogger.info("S3EC_SERVER: Starting Ruby server on port #{settings.port}...") + run! + end +end diff --git a/test-server/ruby-v2-server/config.ru b/test-server/ruby-v2-server/config.ru new file mode 100644 index 00000000..99d3a689 --- /dev/null +++ b/test-server/ruby-v2-server/config.ru @@ -0,0 +1,3 @@ +require_relative 'app' + +run S3ECRubyServer diff --git a/test-server/ruby-v2-server/lib/client_manager.rb b/test-server/ruby-v2-server/lib/client_manager.rb new file mode 100644 index 00000000..3da62b45 --- /dev/null +++ b/test-server/ruby-v2-server/lib/client_manager.rb @@ -0,0 +1,109 @@ +require 'concurrent-ruby' +require 'securerandom' +require 'aws-sdk-s3' +require 'aws-sdk-kms' +require 'openssl' +require 'base64' +require_relative 'logger' + +# Manages S3 Encryption Client instances +class ClientManager + def initialize + @client_cache = Concurrent::Hash.new + @kms_client = Aws::KMS::Client.new(region: 'us-west-2') + S3ECLogger.info("CLIENT_MANAGER: Initialized with KMS client for us-west-2") + end + + # Create a new S3 encryption client and return its ID + def create_client(config) + # Extract all key material types + kms_key_id = config.dig('keyMaterial', 'kmsKeyId') + rsa_key_blob = config.dig('keyMaterial', 'rsaKey') + aes_key_blob = config.dig('keyMaterial', 'aesKey') + inst_file_put = config.dig('instructionFileConfig', 'enableInstructionFilePutObject') + + # Validate that only one key type is provided + key_count = [kms_key_id, rsa_key_blob, aes_key_blob].compact.count + raise 'KeyMaterial must contain exactly one non-null key type' unless key_count == 1 + + # Create S3 encryption client configuration + encryption_config = { + content_encryption_schema: :aes_gcm_no_padding, + envelope_location: inst_file_put ? :instruction_file : :metadata + } + + # Configure based on key type + if kms_key_id + encryption_config[:kms_key_id] = kms_key_id + encryption_config[:kms_client] = @kms_client + encryption_config[:key_wrap_schema] = :kms_context + elsif rsa_key_blob + # Parse RSA private key from PKCS8 format + key_bytes = Base64.decode64(rsa_key_blob) + rsa_key = OpenSSL::PKey::RSA.new(key_bytes) + encryption_config[:encryption_key] = rsa_key + encryption_config[:key_wrap_schema] = :rsa_oaep_sha1 + elsif aes_key_blob + # Extract AES key bytes + key_bytes = Base64.decode64(aes_key_blob) + encryption_config[:encryption_key] = key_bytes + encryption_config[:key_wrap_schema] = :aes_gcm + end + + # Apply legacy settings + encryption_config.tap do |hash| + if !config['enableLegacyWrappingAlgorithms'].nil? || !config['enableLegacyUnauthenticatedModes'].nil? + legacy_modes = config['enableLegacyWrappingAlgorithms'] || config['enableLegacyUnauthenticatedModes'] + # Set security profile based on legacy wrapping algorithms setting + hash[:security_profile] = legacy_modes ? :v2_and_legacy : :v2 + end + end + + # Create the S3 encryption client + # Create the S3 encryption client with retry configuration for throttling + s3_client = Aws::S3::Client.new( + region: 'us-west-2', + retry_mode: 'adaptive', + retry_limit: 5, + retry_backoff: lambda { |c| sleep(2 ** c.retries * 0.3 * rand) } + ) + encryption_client = Aws::S3::EncryptionV2::Client.new( + client: s3_client, + **encryption_config + ) + + # Generate client ID and store in cache + client_id = SecureRandom.uuid + @client_cache[client_id] = encryption_client + + # Log client creation + S3ECLogger.log_client_creation(config, client_id) + S3ECLogger.log_cache_stats(@client_cache.size) + + client_id + end + + # Get a client by ID + def get_client(client_id) + client = @client_cache[client_id] + if client + S3ECLogger.log_client_cache_hit(client_id) + else + S3ECLogger.log_client_cache_miss(client_id) + end + client + end + + # Remove a client from cache (optional cleanup) + def remove_client(client_id) + removed = @client_cache.delete(client_id) + S3ECLogger.info("CLIENT_CACHE: Removed client #{client_id} from cache") if removed + S3ECLogger.log_cache_stats(@client_cache.size) + removed + end + + # Get cache size (for debugging) + def cache_size + @client_cache.size + end +end diff --git a/test-server/ruby-v2-server/lib/error_handlers.rb b/test-server/ruby-v2-server/lib/error_handlers.rb new file mode 100644 index 00000000..234a9a55 --- /dev/null +++ b/test-server/ruby-v2-server/lib/error_handlers.rb @@ -0,0 +1,42 @@ +# Error handling utilities to match Smithy error types +require 'json' + +class ErrorHandlers + # Create a response that matches the GenericServerError type from the Smithy model + # Used for internal server errors + def self.create_generic_server_error(message, status_code = 500) + { + status: status_code, + headers: { 'Content-Type' => 'application/json' }, + body: { + '__type' => 'software.amazon.encryption.s3#GenericServerError', + 'message' => message + }.to_json + } + end + + # Create a response that matches the S3EncryptionClientError type from the Smithy model + # Used for errors thrown by the S3 Encryption Client + def self.create_s3_encryption_client_error(message, status_code = 500) + { + status: status_code, + headers: { 'Content-Type' => 'application/json' }, + body: { + '__type' => 'software.amazon.encryption.s3#S3EncryptionClientError', + 'message' => message + }.to_json + } + end + + # Helper method to send error response in Sinatra + def self.send_generic_server_error(app, message, status_code = 500) + error_response = create_generic_server_error(message, status_code) + app.halt error_response[:status], error_response[:headers], error_response[:body] + end + + # Helper method to send S3EC error response in Sinatra + def self.send_s3_encryption_client_error(app, message, status_code = 500) + error_response = create_s3_encryption_client_error(message, status_code) + app.halt error_response[:status], error_response[:headers], error_response[:body] + end +end diff --git a/test-server/ruby-v2-server/lib/logger.rb b/test-server/ruby-v2-server/lib/logger.rb new file mode 100644 index 00000000..3e820c7f --- /dev/null +++ b/test-server/ruby-v2-server/lib/logger.rb @@ -0,0 +1,105 @@ +require 'logger' +require 'securerandom' + +# Centralized logging utility for the S3EC Ruby server +class S3ECLogger + def self.instance + @instance ||= new + end + + def initialize + @logger = Logger.new(STDOUT) + @logger.level = ENV['LOG_LEVEL'] ? Logger.const_get(ENV['LOG_LEVEL'].upcase) : Logger::INFO + @logger.formatter = proc do |severity, datetime, progname, msg| + "[RUBY TRANSITIONAL #{datetime.strftime('%Y-%m-%d %H:%M:%S')}] #{severity}: #{msg}\n" + end + end + + # Generate a unique request ID for correlation + def self.generate_request_id + SecureRandom.hex(8) + end + + # Request/Response logging + def self.log_request(method, path, headers = {}, request_id = nil) + client_id = headers['HTTP_CLIENTID'] || headers['ClientID'] || 'none' + content_metadata = headers['HTTP_CONTENT_METADATA'] || headers['Content-Metadata'] || 'none' + + instance.logger.info("REQUEST [#{request_id}] #{method} #{path} | ClientID: #{client_id} | Metadata: #{content_metadata}") + end + + def self.log_response(status, request_id = nil, additional_info = "") + info_str = additional_info.empty? ? "" : " | #{additional_info}" + instance.logger.info("RESPONSE [#{request_id}] Status: #{status}#{info_str}") + end + + # Operation-level logging + def self.log_client_creation(config, client_id) + kms_key = config.dig('keyMaterial', 'kmsKeyId') || 'unknown' + legacy_enabled = config['enableLegacyWrappingAlgorithms'] || false + instance.logger.info("CLIENT_CREATION: Created S3EC client #{client_id} | KMS Key: #{kms_key} | Legacy: #{legacy_enabled}") + end + + def self.log_client_cache_hit(client_id) + instance.logger.debug("CACHE_HIT: Found client #{client_id} in cache") + end + + def self.log_client_cache_miss(client_id) + instance.logger.warn("CACHE_MISS: Client #{client_id} not found in cache") + end + + def self.log_cache_stats(cache_size) + instance.logger.debug("CACHE_STATS: Current client cache size: #{cache_size}") + end + + def self.log_s3_operation(operation, bucket, key, encryption_context = {}, additional_info = "") + enc_ctx_str = encryption_context.empty? ? "none" : encryption_context.inspect + info_str = additional_info.empty? ? "" : " | #{additional_info}" + instance.logger.info("S3_OPERATION: #{operation.upcase} s3://#{bucket}/#{key} | EncCtx: #{enc_ctx_str}#{info_str}") + end + + def self.log_metadata_processing(operation, input, output) + instance.logger.debug("METADATA_#{operation.upcase}: Input: #{input.inspect} | Output: #{output.inspect}") + end + + # Enhanced error logging + def self.log_error(error, context = {}, request_id = nil) + error_context = context.empty? ? "" : " | Context: #{context.inspect}" + instance.logger.error("ERROR [#{request_id}] #{error.class}: #{error.message}#{error_context}") + + if error.backtrace && instance.debug? + instance.logger.debug("ERROR_BACKTRACE [#{request_id}]:\n#{error.backtrace.join("\n")}") + end + end + + def self.log_validation_error(field, value, request_id = nil) + instance.logger.warn("VALIDATION_ERROR [#{request_id}] Invalid #{field}: #{value}") + end + + def self.log_aws_error(error, operation, request_id = nil) + instance.logger.error("AWS_ERROR [#{request_id}] #{operation} failed: #{error.class} - #{error.message}") + end + + # Standard logging methods + def self.debug(message) + instance.logger.debug(message) + end + + def self.info(message) + instance.logger.info(message) + end + + def self.warn(message) + instance.logger.warn(message) + end + + def self.error(message) + instance.logger.error(message) + end + + attr_reader :logger + + def debug? + @logger.debug? + end +end diff --git a/test-server/ruby-v2-server/lib/metadata_utils.rb b/test-server/ruby-v2-server/lib/metadata_utils.rb new file mode 100644 index 00000000..72015fcc --- /dev/null +++ b/test-server/ruby-v2-server/lib/metadata_utils.rb @@ -0,0 +1,50 @@ +# Utility class for handling metadata serialization/deserialization +# Matches the format used by Java and Python servers: [key]:[value],[key2]:[value2] +class MetadataUtils + # Convert metadata string to hash + # Input: "[user-defined-enc-ctx-key]:[user-defined-enc-ctx-value],[user-defined-enc-ctx-key-2]:[user-defined-enc-ctx-value-2]" + # Output: {"user-defined-enc-ctx-key" => "user-defined-enc-ctx-value", "user-defined-enc-ctx-key-2" => "user-defined-enc-ctx-value-2"} + def self.string_to_map(metadata_string) + return {} if metadata_string.nil? || metadata_string.empty? + + metadata = {} + entries = metadata_string.split(',') + + entries.each do |entry| + # Split on "]:[" to separate key and value + parts = entry.split(']:[') + if parts.length == 2 + # Remove remaining brackets from start and end + key = parts[0].delete_prefix("[") # Remove first character '[' + value = parts[1].delete_suffix("]") # Remove last character ']' + metadata[key] = value + else + raise "Malformed metadata list entry: #{entry}" + end + end + + metadata + end + + # Convert hash to metadata string + # Input: {"user-defined-enc-ctx-key" => "user-defined-enc-ctx-value", "user-defined-enc-ctx-key-2" => "user-defined-enc-ctx-value-2"} + # Output: "[user-defined-enc-ctx-key]:[user-defined-enc-ctx-value],[user-defined-enc-ctx-key-2]:[user-defined-enc-ctx-value-2]" + def self.map_to_string(metadata_hash) + return '' if metadata_hash.nil? || metadata_hash.empty? + + entries = metadata_hash.map do |key, value| + "[#{key}]:[#{value}]" + end + + entries.join(',') + end + + # Convert metadata hash to array format (for JSON responses) + def self.map_to_array(metadata_hash) + return [] if metadata_hash.nil? || metadata_hash.empty? + + metadata_hash.map do |key, value| + "[#{key}]:[#{value}]" + end + end +end diff --git a/test-server/ruby-v2-server/local-ruby-sdk b/test-server/ruby-v2-server/local-ruby-sdk new file mode 160000 index 00000000..93985e94 --- /dev/null +++ b/test-server/ruby-v2-server/local-ruby-sdk @@ -0,0 +1 @@ +Subproject commit 93985e94bbe8345cc7d615d1cdbcd7516ac16bcd diff --git a/test-server/ruby-v3-server/.bundle/config b/test-server/ruby-v3-server/.bundle/config new file mode 100644 index 00000000..23692288 --- /dev/null +++ b/test-server/ruby-v3-server/.bundle/config @@ -0,0 +1,2 @@ +--- +BUNDLE_PATH: "vendor/bundle" diff --git a/test-server/ruby-v3-server/.duvet/.gitignore b/test-server/ruby-v3-server/.duvet/.gitignore new file mode 100644 index 00000000..93956e36 --- /dev/null +++ b/test-server/ruby-v3-server/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ \ No newline at end of file diff --git a/test-server/ruby-v3-server/.duvet/config.toml b/test-server/ruby-v3-server/.duvet/config.toml new file mode 100644 index 00000000..7a34c0ff --- /dev/null +++ b/test-server/ruby-v3-server/.duvet/config.toml @@ -0,0 +1,33 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "local-ruby-sdk/gems/aws-sdk-s3/lib/**/*.rb" +comment-style = { meta = "##=", content = "##%" } + +[[source]] +pattern = "local-ruby-sdk/gems/aws-sdk-s3/spec/**/*.rb" +comment-style = { meta = "##=", content = "##%" } + +# Include required specifications here +[[specification]] +source = "../specification/s3-encryption/client.md" +[[specification]] +source = "../specification/s3-encryption/decryption.md" +[[specification]] +source = "../specification/s3-encryption/encryption.md" +[[specification]] +source = "../specification/s3-encryption/key-commitment.md" +[[specification]] +source = "../specification/s3-encryption/key-derivation.md" +[[specification]] +source = "../specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "../specification/s3-encryption/data-format/metadata-strategy.md" + + +[report.html] +enabled = true + +# Enable snapshots to prevent requirement coverage regressions +[report.snapshot] +enabled = false diff --git a/test-server/ruby-v3-server/.gitignore b/test-server/ruby-v3-server/.gitignore new file mode 100644 index 00000000..d20a29c0 --- /dev/null +++ b/test-server/ruby-v3-server/.gitignore @@ -0,0 +1,2 @@ +vendor +server.pid \ No newline at end of file diff --git a/test-server/ruby-v3-server/Gemfile b/test-server/ruby-v3-server/Gemfile new file mode 100644 index 00000000..2759a87b --- /dev/null +++ b/test-server/ruby-v3-server/Gemfile @@ -0,0 +1,15 @@ +source 'https://rubygems.org' + +ruby '~> 3.0' + +gem 'sinatra', '~> 3.0' +gem 'puma', '~> 6.0' +gem 'aws-sdk-s3', path: 'local-ruby-sdk/gems/aws-sdk-s3' +gem 'aws-sdk-kms', path: 'local-ruby-sdk/gems/aws-sdk-kms' +gem 'json', '~> 2.0' +gem 'concurrent-ruby', '~> 1.0' +gem 'nokogiri', '~> 1.13' + +group :development do + gem 'rubocop', '~> 1.0' +end diff --git a/test-server/ruby-v3-server/Gemfile.lock b/test-server/ruby-v3-server/Gemfile.lock new file mode 100644 index 00000000..b9f08375 --- /dev/null +++ b/test-server/ruby-v3-server/Gemfile.lock @@ -0,0 +1,111 @@ +PATH + remote: local-ruby-sdk/gems/aws-sdk-kms + specs: + aws-sdk-kms (1.118.0) + aws-sdk-core (~> 3, >= 3.239.1) + aws-sigv4 (~> 1.5) + +PATH + remote: local-ruby-sdk/gems/aws-sdk-s3 + specs: + aws-sdk-s3 (1.206.0) + aws-sdk-core (~> 3, >= 3.234.0) + aws-sdk-kms (~> 1) + aws-sigv4 (~> 1.5) + +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.3) + aws-eventstream (1.4.0) + aws-partitions (1.1180.0) + aws-sdk-core (3.239.2) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) + base64 + bigdecimal + jmespath (~> 1, >= 1.6.1) + logger + aws-sigv4 (1.12.1) + aws-eventstream (~> 1, >= 1.0.2) + base64 (0.3.0) + bigdecimal (3.3.1) + bigdecimal (3.3.1-java) + concurrent-ruby (1.3.5) + jmespath (1.6.2) + json (2.13.2) + json (2.13.2-java) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + logger (1.7.0) + mustermann (3.0.4) + ruby2_keywords (~> 0.0.1) + nio4r (2.7.4) + nio4r (2.7.4-java) + nokogiri (1.18.10-arm64-darwin) + racc (~> 1.4) + nokogiri (1.18.10-java) + racc (~> 1.4) + parallel (1.27.0) + parser (3.3.9.0) + ast (~> 2.4.1) + racc + prism (1.5.1) + puma (6.6.1) + nio4r (~> 2.0) + puma (6.6.1-java) + nio4r (~> 2.0) + racc (1.8.1) + racc (1.8.1-java) + rack (2.2.17) + rack-protection (3.2.0) + base64 (>= 0.1.0) + rack (~> 2.2, >= 2.2.4) + rainbow (3.1.1) + regexp_parser (2.11.3) + rubocop (1.80.2) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.46.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.46.0) + parser (>= 3.3.7.2) + prism (~> 1.4) + ruby-progressbar (1.13.0) + ruby2_keywords (0.0.5) + sinatra (3.2.0) + mustermann (~> 3.0) + rack (~> 2.2, >= 2.2.4) + rack-protection (= 3.2.0) + tilt (~> 2.0) + tilt (2.6.1) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.1.0) + +PLATFORMS + arm64-darwin-24 + universal-java-21 + +DEPENDENCIES + aws-sdk-kms! + aws-sdk-s3! + concurrent-ruby (~> 1.0) + json (~> 2.0) + nokogiri (~> 1.13) + puma (~> 6.0) + rubocop (~> 1.0) + sinatra (~> 3.0) + +RUBY VERSION + ruby 3.4.5p51 + +BUNDLED WITH + 2.6.9 diff --git a/test-server/ruby-v3-server/Makefile b/test-server/ruby-v3-server/Makefile new file mode 100644 index 00000000..331abac5 --- /dev/null +++ b/test-server/ruby-v3-server/Makefile @@ -0,0 +1,43 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: build-server start-server stop-server wait-for-server + +PID_FILE := server.pid +PORT := 8092 + +build-server: + @echo "Building Ruby V3 server..." + bundle install + +start-server: + @if [ -f $(PID_FILE) ]; then \ + echo "❌ Error: Server already running. Stop before starting."; \ + exit 1; \ + fi; + @echo "Starting Ruby V3 server..." + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + PORT=$(PORT) bundle exec ruby app.rb > server.log 2>&1 & echo $$! > $(PID_FILE) + @echo "Ruby V3 server starting..." + +stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true + @if [ -f $(PID_FILE) ]; then \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ + fi + @rm -f server.log + @echo "Server stopped" + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT) + +duvet: + duvet report + +view-report-mac: + open .duvet/reports/report.html diff --git a/test-server/ruby-v3-server/README.md b/test-server/ruby-v3-server/README.md new file mode 100644 index 00000000..0c27b3d8 --- /dev/null +++ b/test-server/ruby-v3-server/README.md @@ -0,0 +1,74 @@ +# Ruby S3 Encryption Client Test Server + +This is a Ruby implementation of the S3 Encryption Client test server +that provides an invariant interface around the S3 Encryption Client v3. +It's designed to work alongside other implementations of test servers for cross-language compatibility testing. + +## Overview + +The server provides a REST API that wraps the AWS S3 Encryption Client v3, +allowing tests to verify that all language implementations behave consistently. + +## Endpoints + +- `POST /client` - Create a new S3 encryption client instance +- `PUT /object/{bucket}/{key}` - Encrypt and store an object +- `GET /object/{bucket}/{key}` - Retrieve and decrypt an object +- `GET /health` - Health check endpoint + +## Configuration + +The server runs on port **8092** by default. + +## Setup + +1. Install Ruby 3.x +2. Install dependencies: + + ```bash + cd test-server/ruby-v2-server + bundle install + ``` + +3. Set up AWS credentials (via AWS CLI, environment variables, or IAM roles) + +4. Start the server: + + ```bash + ruby app.rb + # or using Rack + bundle exec rackup -p 8092 + ``` + +## Usage + +The server is designed to be used by the Java test suite in `test-server/java-tests/`. +The tests will automatically discover and use this server for cross-language compatibility testing. + +### Environment Variables + +- `TEST_SERVER_KMS_KEY_ARN` - KMS key ARN for encryption (defaults to test key) +- `TEST_SERVER_S3_BUCKET` - S3 bucket for testing (defaults to test bucket) + +## Architecture + +- `app.rb` - Main Sinatra application +- `lib/client_manager.rb` - Manages S3 encryption client instances +- `lib/metadata_utils.rb` - Handles metadata serialization/deserialization +- `lib/error_handlers.rb` - Smithy-compliant error responses + +## Error Handling + +The server returns errors in the format expected by the Smithy model: + +- `GenericServerError` - Internal server errors +- `S3EncryptionClientError` - Errors from the S3 Encryption Client + +## Compatibility + +This server is compatible with: + +- S3 Encryption Client v3 +- Legacy v1 clients (when `enableLegacyWrappingAlgorithms` is true) +- Legacy v2 clients (when `???` is true) +- Cross-language testing with other implementations diff --git a/test-server/ruby-v3-server/app.rb b/test-server/ruby-v3-server/app.rb new file mode 100644 index 00000000..80ac972f --- /dev/null +++ b/test-server/ruby-v3-server/app.rb @@ -0,0 +1,241 @@ +require 'sinatra' +require 'json' +require_relative 'lib/client_manager' +require_relative 'lib/metadata_utils' +require_relative 'lib/error_handlers' +require_relative 'lib/logger' + +# See: https://github.com/ruby/openssl/issues/949 +Aws.use_bundled_cert! + +class S3ECRubyServer < Sinatra::Base + configure do + set :port, ENV['PORT'] || 8092 + set :bind, '0.0.0.0' + set :show_exceptions, false + set :raise_errors, false + end + + def initialize + super + @client_manager = ClientManager.new + S3ECLogger.info("S3EC_SERVER: Ruby V3 server initialized on port #{settings.port}") + end + + # Request logging middleware + before do + @request_id = S3ECLogger.generate_request_id + S3ECLogger.log_request(request.request_method, request.path_info, request.env, @request_id) + end + + # Response logging middleware + after do + S3ECLogger.log_response(response.status, @request_id) + end + + # Health check endpoint + get '/health' do + content_type :json + { status: 'OK', server: 'Ruby V3 S3EC Test Server', port: settings.port.to_i }.to_json + end + + # POST /client - Create S3 encryption client + post '/client' do + begin + S3ECLogger.debug("CLIENT_ENDPOINT [#{@request_id}]: Processing client creation request") + + # Parse request body + request_body = request.body.read + S3ECLogger.debug("CLIENT_ENDPOINT [#{@request_id}]: Request body size: #{request_body.length} bytes") + + parsed_data = JSON.parse(request_body) + config = parsed_data['config'] || {} + + S3ECLogger.debug("CLIENT_ENDPOINT [#{@request_id}]: Parsed config: #{config.inspect}") + + # Create client using client manager + client_id = @client_manager.create_client(config) + + S3ECLogger.info("CLIENT_ENDPOINT [#{@request_id}]: Successfully created client #{client_id}") + + # Return client ID + content_type :json + { clientId: client_id }.to_json + + rescue JSON::ParserError => e + S3ECLogger.log_error(e, { endpoint: '/client', operation: 'JSON parsing' }, @request_id) + ErrorHandlers.send_generic_server_error(self, "Invalid JSON in request body", 400) + rescue => e + S3ECLogger.log_error(e, { endpoint: '/client', operation: 'client creation', config: config }, @request_id) + ErrorHandlers.send_s3_encryption_client_error(self, "Failed to create client: #{e.message}") + end + end + + # PUT /object/{bucket}/{key} - Encrypt and put object + put '/object/:bucket/:key' do + bucket = params[:bucket] + key = params[:key] + client_id = request.env['HTTP_CLIENTID'] + + begin + S3ECLogger.debug("PUT_ENDPOINT [#{@request_id}]: Processing PUT request for s3://#{bucket}/#{key}") + + # Validate client ID + unless client_id + S3ECLogger.log_validation_error('ClientID', 'missing', @request_id) + ErrorHandlers.send_generic_server_error(self, "ClientID header is required", 400) + end + + # Get client from cache + client = @client_manager.get_client(client_id) + unless client + S3ECLogger.log_validation_error('ClientID', client_id, @request_id) + ErrorHandlers.send_generic_server_error(self, "No client found for ClientID: #{client_id}", 404) + end + + # Get request body + body = request.body.read + S3ECLogger.debug("PUT_ENDPOINT [#{@request_id}]: Request body size: #{body.length} bytes") + + # Parse metadata from header + metadata_header = request.env['HTTP_CONTENT_METADATA'] || '' + encryption_context = MetadataUtils.string_to_map(metadata_header) + S3ECLogger.log_metadata_processing('parse', metadata_header, encryption_context) + + # Prepare S3 put_object parameters + put_params = { + bucket: bucket, + key: key, + body: body + } + + # Add encryption context if present + put_params[:kms_encryption_context] = encryption_context unless encryption_context.empty? + + # Log S3 operation + S3ECLogger.log_s3_operation('put', bucket, key, encryption_context, "ClientID: #{client_id}, BodySize: #{body.length}") + + # Make the put_object request + response = client.put_object(put_params) + + S3ECLogger.info("PUT_ENDPOINT [#{@request_id}]: Successfully put object s3://#{bucket}/#{key}") + + # Prepare response metadata + response_metadata = MetadataUtils.map_to_array(encryption_context) + S3ECLogger.log_metadata_processing('response', encryption_context, response_metadata) + + # Return response matching Smithy model + content_type :json + { + bucket: bucket, + key: key, + metadata: response_metadata + }.to_json + + rescue Aws::S3::EncryptionV2::Errors::EncryptionError, Aws::S3::EncryptionV3::Errors::EncryptionError => e + S3ECLogger.log_error(e, { endpoint: '/put', error_category: 'EncryptionError' }, @request_id) + ErrorHandlers.send_s3_encryption_client_error(self, e.message) + rescue StandardError => e + # Handle generic server errors (return as GenericServerError) + S3ECLogger.log_error(e, { endpoint: '/put', error_category: 'generic_server' }, @request_id) + status_code = e.respond_to?(:code) ? e.code : 500 + ErrorHandlers.send_generic_server_error(self, e.message, status_code) + end + end + + # GET /object/{bucket}/{key} - Get and decrypt object + get '/object/:bucket/:key' do + bucket = params[:bucket] + key = params[:key] + client_id = request.env['HTTP_CLIENTID'] + + begin + S3ECLogger.debug("GET_ENDPOINT [#{@request_id}]: Processing GET request for s3://#{bucket}/#{key}") + + # Validate client ID + unless client_id + S3ECLogger.log_validation_error('ClientID', 'missing', @request_id) + ErrorHandlers.send_generic_server_error(self, "ClientID header is required", 400) + end + + # Get client from cache + client = @client_manager.get_client(client_id) + unless client + S3ECLogger.log_validation_error('ClientID', client_id, @request_id) + ErrorHandlers.send_generic_server_error(self, "No client found for ClientID: #{client_id}", 404) + end + + # Parse metadata from header + metadata_header = request.env['HTTP_CONTENT_METADATA'] || '' + encryption_context = MetadataUtils.string_to_map(metadata_header) + S3ECLogger.log_metadata_processing('parse', metadata_header, encryption_context) + + # Prepare S3 get_object parameters + get_params = { + bucket: bucket, + key: key + } + + # Add custom instruction file suffix if present + instruction_file_suffix = request.env['HTTP_INSTRUCTIONFILESUFFIX'] + if instruction_file_suffix && !instruction_file_suffix.empty? + get_params[:envelope_location] = :instruction_file + get_params[:instruction_file_suffix] = instruction_file_suffix + S3ECLogger.debug("GET_ENDPOINT [#{@request_id}]: Using custom instruction file suffix: #{instruction_file_suffix}") + elsif !encryption_context.empty? + get_params[:kms_encryption_context] = encryption_context + end + + # Log S3 operation + S3ECLogger.log_s3_operation('get', bucket, key, encryption_context, "ClientID: #{client_id}") + + # Make the get_object request + response = client.get_object(get_params) + + # Extract body and metadata + body = response.body.read + metadata = response.metadata || {} + + S3ECLogger.info("GET_ENDPOINT [#{@request_id}]: Successfully got object s3://#{bucket}/#{key}, BodySize: #{body.length}") + + # Set Content-Metadata header in response + metadata_str = MetadataUtils.map_to_string(metadata) + S3ECLogger.log_metadata_processing('response', metadata, metadata_str) + + headers['Content-Metadata'] = metadata_str unless metadata_str.empty? + + # Return the body as response + content_type 'application/octet-stream' + body + + rescue Aws::S3::EncryptionV2::Errors::DecryptionError, Aws::S3::EncryptionV3::Errors::DecryptionError => e + S3ECLogger.log_error(e, { endpoint: '/get', error_category: 'DecryptionError' }, @request_id) + ErrorHandlers.send_s3_encryption_client_error(self, e.message) + rescue StandardError => e + # Handle generic server errors (return as GenericServerError) + S3ECLogger.log_error(e, { endpoint: '/get', error_category: 'generic_server' }, @request_id) + status_code = e.respond_to?(:code) ? e.code : 500 + ErrorHandlers.send_generic_server_error(self, e.message, status_code) + end + end + + # Global error handler + error do + error = env['sinatra.error'] + context = { + endpoint: request.path_info, + method: request.request_method, + params: params, + error_type: 'global_error_handler' + } + + S3ECLogger.log_error(error, context, @request_id) + ErrorHandlers.send_generic_server_error(self, "Internal server error: #{error.message}") + end + + # Start server when run directly + if __FILE__ == $0 + S3ECLogger.info("S3EC_SERVER: Starting Ruby server on port #{settings.port}...") + run! + end +end diff --git a/test-server/ruby-v3-server/config.ru b/test-server/ruby-v3-server/config.ru new file mode 100644 index 00000000..99d3a689 --- /dev/null +++ b/test-server/ruby-v3-server/config.ru @@ -0,0 +1,3 @@ +require_relative 'app' + +run S3ECRubyServer diff --git a/test-server/ruby-v3-server/lib/client_manager.rb b/test-server/ruby-v3-server/lib/client_manager.rb new file mode 100644 index 00000000..5ee3f1ec --- /dev/null +++ b/test-server/ruby-v3-server/lib/client_manager.rb @@ -0,0 +1,134 @@ +require 'concurrent-ruby' +require 'securerandom' +require 'aws-sdk-s3' +require 'aws-sdk-kms' +require 'openssl' +require 'base64' +require_relative 'logger' + +# Manages S3 Encryption Client instances +class ClientManager + def initialize + @client_cache = Concurrent::Hash.new + @kms_client = Aws::KMS::Client.new(region: 'us-west-2') + S3ECLogger.info("CLIENT_MANAGER: Initialized with KMS client for us-west-2") + end + + # Create a new S3 encryption client and return its ID + def create_client(config) + # Extract all key material types + kms_key_id = config.dig('keyMaterial', 'kmsKeyId') + rsa_key_blob = config.dig('keyMaterial', 'rsaKey') + aes_key_blob = config.dig('keyMaterial', 'aesKey') + inst_file_put = config.dig('instructionFileConfig', 'enableInstructionFilePutObject') + content_alg = config.dig('encryptionAlgorithm') + + # Validate that only one key type is provided + key_count = [kms_key_id, rsa_key_blob, aes_key_blob].compact.count + raise 'KeyMaterial must contain exactly one non-null key type' unless key_count == 1 + + # translate between canonical AlgSuite and Ruby symbols + if content_alg.nil? || content_alg == 'ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY' + content_alg = :alg_aes_256_gcm_hkdf_sha512_commit_key + elsif content_alg == 'ALG_AES_256_GCM_IV12_TAG16_NO_KDF' + content_alg = :aes_gcm_no_padding + else + raise 'Unknown content encryption algorithm provided: ' + content_alg + end + + # Create S3 encryption client configuration + encryption_config = { + envelope_location: inst_file_put ? :instruction_file : :metadata, + content_encryption_schema: content_alg + } + + # Configure based on key type + if kms_key_id + encryption_config[:kms_key_id] = kms_key_id + encryption_config[:kms_client] = @kms_client + encryption_config[:key_wrap_schema] = :kms_context + elsif rsa_key_blob + # Parse RSA private key from PKCS8 format + key_bytes = Base64.decode64(rsa_key_blob) + rsa_key = OpenSSL::PKey::RSA.new(key_bytes) + encryption_config[:encryption_key] = rsa_key + encryption_config[:key_wrap_schema] = :rsa_oaep_sha1 + elsif aes_key_blob + # Extract AES key bytes + key_bytes = Base64.decode64(aes_key_blob) + encryption_config[:encryption_key] = key_bytes + encryption_config[:key_wrap_schema] = :aes_gcm + end + + # Apply additional configuration + encryption_config.tap do |hash| + if !config['commitmentPolicy'].nil? + hash[:commitment_policy] = case config['commitmentPolicy'] + when 'FORBID_ENCRYPT_ALLOW_DECRYPT' + :forbid_encrypt_allow_decrypt + when 'REQUIRE_ENCRYPT_ALLOW_DECRYPT' + :require_encrypt_allow_decrypt + when 'REQUIRE_ENCRYPT_REQUIRE_DECRYPT' + :require_encrypt_require_decrypt + else + raise "Unsupported commitment_policy " + config['commitmentPolicy'] + end + if config['commitmentPolicy'] == 'FORBID_ENCRYPT_ALLOW_DECRYPT' && config['encryptionAlgorithm'].nil? + hash[:content_encryption_schema] = :aes_gcm_no_padding + end + end + if !config['enableLegacyWrappingAlgorithms'].nil? || !config['enableLegacyUnauthenticatedModes'].nil? + legacy_modes = config['enableLegacyWrappingAlgorithms'] || config['enableLegacyUnauthenticatedModes'] + # Set security profile based on legacy wrapping algorithms setting + hash[:security_profile] = legacy_modes ? :v3_and_legacy : :v3 + end + end + + # Create the S3 encryption client + # Create the S3 encryption client with retry configuration for throttling + s3_client = Aws::S3::Client.new( + region: 'us-west-2', + retry_mode: 'adaptive', + retry_limit: 5, + retry_backoff: lambda { |c| sleep(2 ** c.retries * 0.3 * rand) } + ) + encryption_client = Aws::S3::EncryptionV3::Client.new( + client: s3_client, + **encryption_config + ) + + # Generate client ID and store in cache + client_id = SecureRandom.uuid + @client_cache[client_id] = encryption_client + + # Log client creation + S3ECLogger.log_client_creation(config, client_id) + S3ECLogger.log_cache_stats(@client_cache.size) + + client_id + end + + # Get a client by ID + def get_client(client_id) + client = @client_cache[client_id] + if client + S3ECLogger.log_client_cache_hit(client_id) + else + S3ECLogger.log_client_cache_miss(client_id) + end + client + end + + # Remove a client from cache (optional cleanup) + def remove_client(client_id) + removed = @client_cache.delete(client_id) + S3ECLogger.info("CLIENT_CACHE: Removed client #{client_id} from cache") if removed + S3ECLogger.log_cache_stats(@client_cache.size) + removed + end + + # Get cache size (for debugging) + def cache_size + @client_cache.size + end +end diff --git a/test-server/ruby-v3-server/lib/error_handlers.rb b/test-server/ruby-v3-server/lib/error_handlers.rb new file mode 100644 index 00000000..234a9a55 --- /dev/null +++ b/test-server/ruby-v3-server/lib/error_handlers.rb @@ -0,0 +1,42 @@ +# Error handling utilities to match Smithy error types +require 'json' + +class ErrorHandlers + # Create a response that matches the GenericServerError type from the Smithy model + # Used for internal server errors + def self.create_generic_server_error(message, status_code = 500) + { + status: status_code, + headers: { 'Content-Type' => 'application/json' }, + body: { + '__type' => 'software.amazon.encryption.s3#GenericServerError', + 'message' => message + }.to_json + } + end + + # Create a response that matches the S3EncryptionClientError type from the Smithy model + # Used for errors thrown by the S3 Encryption Client + def self.create_s3_encryption_client_error(message, status_code = 500) + { + status: status_code, + headers: { 'Content-Type' => 'application/json' }, + body: { + '__type' => 'software.amazon.encryption.s3#S3EncryptionClientError', + 'message' => message + }.to_json + } + end + + # Helper method to send error response in Sinatra + def self.send_generic_server_error(app, message, status_code = 500) + error_response = create_generic_server_error(message, status_code) + app.halt error_response[:status], error_response[:headers], error_response[:body] + end + + # Helper method to send S3EC error response in Sinatra + def self.send_s3_encryption_client_error(app, message, status_code = 500) + error_response = create_s3_encryption_client_error(message, status_code) + app.halt error_response[:status], error_response[:headers], error_response[:body] + end +end diff --git a/test-server/ruby-v3-server/lib/logger.rb b/test-server/ruby-v3-server/lib/logger.rb new file mode 100644 index 00000000..df8ad9db --- /dev/null +++ b/test-server/ruby-v3-server/lib/logger.rb @@ -0,0 +1,105 @@ +require 'logger' +require 'securerandom' + +# Centralized logging utility for the S3EC Ruby server +class S3ECLogger + def self.instance + @instance ||= new + end + + def initialize + @logger = Logger.new(STDOUT) + @logger.level = ENV['LOG_LEVEL'] ? Logger.const_get(ENV['LOG_LEVEL'].upcase) : Logger::INFO + @logger.formatter = proc do |severity, datetime, progname, msg| + "[RUBY IMPROVED #{datetime.strftime('%Y-%m-%d %H:%M:%S')}] #{severity}: #{msg}\n" + end + end + + # Generate a unique request ID for correlation + def self.generate_request_id + SecureRandom.hex(8) + end + + # Request/Response logging + def self.log_request(method, path, headers = {}, request_id = nil) + client_id = headers['HTTP_CLIENTID'] || headers['ClientID'] || 'none' + content_metadata = headers['HTTP_CONTENT_METADATA'] || headers['Content-Metadata'] || 'none' + + instance.logger.info("REQUEST [#{request_id}] #{method} #{path} | ClientID: #{client_id} | Metadata: #{content_metadata}") + end + + def self.log_response(status, request_id = nil, additional_info = "") + info_str = additional_info.empty? ? "" : " | #{additional_info}" + instance.logger.info("RESPONSE [#{request_id}] Status: #{status}#{info_str}") + end + + # Operation-level logging + def self.log_client_creation(config, client_id) + kms_key = config.dig('keyMaterial', 'kmsKeyId') || 'unknown' + legacy_enabled = config['enableLegacyWrappingAlgorithms'] || false + instance.logger.info("CLIENT_CREATION: Created S3EC client #{client_id} | KMS Key: #{kms_key} | Legacy: #{legacy_enabled}") + end + + def self.log_client_cache_hit(client_id) + instance.logger.debug("CACHE_HIT: Found client #{client_id} in cache") + end + + def self.log_client_cache_miss(client_id) + instance.logger.warn("CACHE_MISS: Client #{client_id} not found in cache") + end + + def self.log_cache_stats(cache_size) + instance.logger.debug("CACHE_STATS: Current client cache size: #{cache_size}") + end + + def self.log_s3_operation(operation, bucket, key, encryption_context = {}, additional_info = "") + enc_ctx_str = encryption_context.empty? ? "none" : encryption_context.inspect + info_str = additional_info.empty? ? "" : " | #{additional_info}" + instance.logger.info("S3_OPERATION: #{operation.upcase} s3://#{bucket}/#{key} | EncCtx: #{enc_ctx_str}#{info_str}") + end + + def self.log_metadata_processing(operation, input, output) + instance.logger.debug("METADATA_#{operation.upcase}: Input: #{input.inspect} | Output: #{output.inspect}") + end + + # Enhanced error logging + def self.log_error(error, context = {}, request_id = nil) + error_context = context.empty? ? "" : " | Context: #{context.inspect}" + instance.logger.error("ERROR [#{request_id}] #{error.class}: #{error.message}#{error_context}") + + if error.backtrace && instance.debug? + instance.logger.debug("ERROR_BACKTRACE [#{request_id}]:\n#{error.backtrace.join("\n")}") + end + end + + def self.log_validation_error(field, value, request_id = nil) + instance.logger.warn("VALIDATION_ERROR [#{request_id}] Invalid #{field}: #{value}") + end + + def self.log_aws_error(error, operation, request_id = nil) + instance.logger.error("AWS_ERROR [#{request_id}] #{operation} failed: #{error.class} - #{error.message}") + end + + # Standard logging methods + def self.debug(message) + instance.logger.debug(message) + end + + def self.info(message) + instance.logger.info(message) + end + + def self.warn(message) + instance.logger.warn(message) + end + + def self.error(message) + instance.logger.error(message) + end + + attr_reader :logger + + def debug? + @logger.debug? + end +end diff --git a/test-server/ruby-v3-server/lib/metadata_utils.rb b/test-server/ruby-v3-server/lib/metadata_utils.rb new file mode 100644 index 00000000..72015fcc --- /dev/null +++ b/test-server/ruby-v3-server/lib/metadata_utils.rb @@ -0,0 +1,50 @@ +# Utility class for handling metadata serialization/deserialization +# Matches the format used by Java and Python servers: [key]:[value],[key2]:[value2] +class MetadataUtils + # Convert metadata string to hash + # Input: "[user-defined-enc-ctx-key]:[user-defined-enc-ctx-value],[user-defined-enc-ctx-key-2]:[user-defined-enc-ctx-value-2]" + # Output: {"user-defined-enc-ctx-key" => "user-defined-enc-ctx-value", "user-defined-enc-ctx-key-2" => "user-defined-enc-ctx-value-2"} + def self.string_to_map(metadata_string) + return {} if metadata_string.nil? || metadata_string.empty? + + metadata = {} + entries = metadata_string.split(',') + + entries.each do |entry| + # Split on "]:[" to separate key and value + parts = entry.split(']:[') + if parts.length == 2 + # Remove remaining brackets from start and end + key = parts[0].delete_prefix("[") # Remove first character '[' + value = parts[1].delete_suffix("]") # Remove last character ']' + metadata[key] = value + else + raise "Malformed metadata list entry: #{entry}" + end + end + + metadata + end + + # Convert hash to metadata string + # Input: {"user-defined-enc-ctx-key" => "user-defined-enc-ctx-value", "user-defined-enc-ctx-key-2" => "user-defined-enc-ctx-value-2"} + # Output: "[user-defined-enc-ctx-key]:[user-defined-enc-ctx-value],[user-defined-enc-ctx-key-2]:[user-defined-enc-ctx-value-2]" + def self.map_to_string(metadata_hash) + return '' if metadata_hash.nil? || metadata_hash.empty? + + entries = metadata_hash.map do |key, value| + "[#{key}]:[#{value}]" + end + + entries.join(',') + end + + # Convert metadata hash to array format (for JSON responses) + def self.map_to_array(metadata_hash) + return [] if metadata_hash.nil? || metadata_hash.empty? + + metadata_hash.map do |key, value| + "[#{key}]:[#{value}]" + end + end +end diff --git a/test-server/ruby-v3-server/local-ruby-sdk b/test-server/ruby-v3-server/local-ruby-sdk new file mode 160000 index 00000000..93985e94 --- /dev/null +++ b/test-server/ruby-v3-server/local-ruby-sdk @@ -0,0 +1 @@ +Subproject commit 93985e94bbe8345cc7d615d1cdbcd7516ac16bcd diff --git a/test-server/spec-compliance-dashboard/.gitignore b/test-server/spec-compliance-dashboard/.gitignore new file mode 100644 index 00000000..c9e1b5bb --- /dev/null +++ b/test-server/spec-compliance-dashboard/.gitignore @@ -0,0 +1 @@ +compliance_homepage.html \ No newline at end of file diff --git a/test-server/spec-compliance-dashboard/generate_compliance_dashboard.py b/test-server/spec-compliance-dashboard/generate_compliance_dashboard.py new file mode 100644 index 00000000..d19f6c6e --- /dev/null +++ b/test-server/spec-compliance-dashboard/generate_compliance_dashboard.py @@ -0,0 +1,1049 @@ +#!/usr/bin/env python3 +""" +Self-contained script to generate compliance dashboard and all server reports. +Automatically discovers servers with .duvet/reports/report.html files and generates +individual reports using the enhanced report-based format with deep links, source traceability, +copy buttons, and comprehensive statistics. +""" + +import json +import re +import os +from pathlib import Path +from datetime import datetime + + +def parse_report_html(report_file_path): + """Parse the report.html file and extract specification data.""" + with open(report_file_path, "r", encoding="utf-8") as f: + content = f.read() + + # Extract JSON from script tag with id="result" + start_marker = '" + + start_idx = content.find(start_marker) + if start_idx == -1: + raise ValueError("No result script tag found in HTML") + + start_idx += len(start_marker) + end_idx = content.find(end_marker, start_idx) + if end_idx == -1: + raise ValueError("No closing script tag found") + + json_content = content[start_idx:end_idx] + data = json.loads(json_content) + + # Convert report.html JSON structure to match snapshot structure + return convert_report_to_specifications(data) + + +def convert_report_to_specifications(data): + """Convert duvet report.html JSON structure to match snapshot structure.""" + specifications = {} + + for spec_path, spec in (data.get("specifications", {})).items(): + spec_data = { + "title": spec.get("title", "Unknown"), + "spec_path": spec_path, # Store the original spec path + "sections": {}, + } + + # Process sections - sections is a list, not a dict + for section in spec.get("sections", []): + section_data = { + "title": section.get("title", "Unknown"), + "section_id": section.get("id", "unknown"), # Store the section ID + "requirements": [], + } + + # Process requirements for this section + for req_id in section.get("requirements", []): + # Get annotation data + annotation = None + if "annotations" in data and isinstance(data["annotations"], list): + # annotations is a list indexed by req_id + if req_id < len(data["annotations"]): + annotation = data["annotations"][req_id] + + # Get status data + status = None + if "statuses" in data and isinstance(data["statuses"], dict): + status = data["statuses"].get(str(req_id)) + + if annotation and status: + # Parse status indicators (matching snapshot logic) + has_implementation = bool( + status.get("citation") + ) # Only citation counts as implementation + has_test = bool(status.get("test")) + has_exception = bool(status.get("exception")) + has_implication = bool(status.get("implication")) + has_partial_coverage = bool(status.get("incomplete")) + + # Determine completion status (matching snapshot rules exactly) + is_complete = ( + (has_implementation and has_test) or has_exception or has_implication + ) and not has_partial_coverage # Partial coverage means not complete + + # Collect related annotations for detailed status + related_sources = [] + if "related" in status: + for related_id in status["related"]: + if related_id < len(data["annotations"]): + related_annotation = data["annotations"][related_id] + source = related_annotation.get("source", "") + line = related_annotation.get("line", "") + annotation_type = related_annotation.get("type", "CITATION") + if source: + source_info = { + "source": source, + "line": line, + "type": annotation_type, + } + related_sources.append(source_info) + + requirement = { + "text": annotation.get("comment", "No comment available"), + "has_implementation": has_implementation, + "has_test": has_test, + "has_exception": has_exception, + "has_implication": has_implication, + "has_partial_coverage": has_partial_coverage, + "is_complete": is_complete, + "related_sources": related_sources, + } + + section_data["requirements"].append(requirement) + elif req_id < len(data.get("annotations", [])): + # Fallback: create requirement with basic info + annotation = data["annotations"][req_id] + requirement = { + "text": annotation.get("comment", f"Requirement {req_id}"), + "has_implementation": False, + "has_test": False, + "has_exception": False, + "has_implication": False, + "is_complete": False, + "related_sources": [], + } + section_data["requirements"].append(requirement) + + spec_data["sections"][section.get("title", "Unknown")] = section_data + + specifications[spec.get("title", "Unknown")] = spec_data + + return specifications + + +def get_spec_status(spec_data): + """Determine the overall status of a specification based on all its sections.""" + sections = spec_data.get("sections", {}) + + if not sections: + return "✅" # No sections means complete + + # Get status of each section + section_statuses = [] + for section_data in sections.values(): + requirements = section_data.get("requirements", []) + if not requirements: + section_statuses.append("✅") # Empty section is complete + else: + complete_reqs = sum(1 for req in requirements if req["is_complete"]) + total_reqs = len(requirements) + + if complete_reqs == total_reqs: + section_statuses.append("✅") # All requirements complete + elif complete_reqs > 0: + section_statuses.append("🟡") # Some requirements complete + else: + section_statuses.append("❌") # No requirements complete + + # Apply the corrected logic based on section statuses: + if all(status == "✅" for status in section_statuses): + return "✅" # Green check if all sections are green + elif any(status in ["✅", "🟡"] for status in section_statuses): + return "🟡" # Yellow if any section is green or yellow + else: + return "❌" # Red X if all sections are red X + + +def get_requirement_status(requirement): + """Get the status emoji for a single requirement.""" + if requirement["is_complete"]: + return "✅" + elif requirement.get("has_partial_coverage", False): + return "🟡" # Partial coverage - incomplete + elif requirement["has_implementation"] and requirement["related_sources"]: + return "🟡" # Has implementation but no test + else: + return "❌" # No implementation + + +def format_requirement_text(text): + """Format requirement text to style status metadata lines.""" + lines = text.split("\n") + formatted_lines = [] + + for line in lines: + # Check if line contains status metadata + if line.strip().startswith("Status:"): + formatted_lines.append(f'{line}') + else: + formatted_lines.append(line) + + return "\n".join(formatted_lines) + + +def calculate_summary_statistics(specifications): + """Calculate summary statistics for all specifications.""" + total_sections = 0 + complete_sections = 0 + total_requirements = 0 + complete_requirements = 0 + + # Count requirements by implementation type + no_implementation = 0 + implementation_only = 0 + test_only = 0 + implementation_and_test = 0 + exception_count = 0 + implication_count = 0 + partial_coverage_count = 0 + + for spec_data in specifications.values(): + sections = spec_data.get("sections", {}) + total_sections += len(sections) + + for section_data in sections.values(): + requirements = section_data.get("requirements", []) + total_requirements += len(requirements) + + # Count complete requirements + section_complete_reqs = sum(1 for req in requirements if req["is_complete"]) + complete_requirements += section_complete_reqs + + # A section is complete if all its requirements are complete + if requirements and section_complete_reqs == len(requirements): + complete_sections += 1 + elif not requirements: # Empty section is considered complete + complete_sections += 1 + + # Count requirements by implementation type + for req in requirements: + if req["has_exception"]: + exception_count += 1 + elif req["has_implication"]: + implication_count += 1 + elif ( + req["has_implementation"] + and req["has_test"] + and not req.get("has_partial_coverage", False) + ): + implementation_and_test += 1 + elif req["has_implementation"] and not req.get("has_partial_coverage", False): + implementation_only += 1 + elif req["has_test"] and not req.get("has_partial_coverage", False): + test_only += 1 + else: + # Partial coverage gets counted as no implementation + no_implementation += 1 + + return { + "total_sections": total_sections, + "complete_sections": complete_sections, + "total_requirements": total_requirements, + "complete_requirements": complete_requirements, + "no_implementation": no_implementation, + "implementation_only": implementation_only, + "test_only": test_only, + "implementation_and_test": implementation_and_test, + "exception_count": exception_count, + "implication_count": implication_count, + "partial_coverage_count": partial_coverage_count, + } + + +def url_encode_spec_path(spec_path): + """URL encode the spec path for use in duvet report URLs.""" + import urllib.parse + + return urllib.parse.quote(spec_path, safe="") + + +def generate_spec_url(duvet_report_path, spec_path): + """Generate URL to a specific specification in the duvet report.""" + encoded_path = url_encode_spec_path(spec_path) + return f"{duvet_report_path}#/spec/{encoded_path}" + + +def generate_section_url(duvet_report_path, spec_path, section_id): + """Generate URL to a specific section in the duvet report.""" + encoded_path = url_encode_spec_path(spec_path) + return f"{duvet_report_path}#/spec/{encoded_path}/{section_id}" + + +def generate_github_url(source_path, line_number=None, github_base_url=None): + """Generate GitHub URL for a source file.""" + if not github_base_url: + return None + + # Convert local path to GitHub path + # Remove local-go-s3ec/ prefix if present + if source_path.startswith("local-go-s3ec/"): + github_path = source_path[len("local-go-s3ec/") :] + else: + github_path = source_path + + url = f"{github_base_url}/{github_path}" + if line_number: + url += f"#L{line_number}" + + return url + + +def load_template(template_path): + """Load a template file.""" + with open(template_path, "r", encoding="utf-8") as f: + return f.read() + + +def generate_enhanced_html_report(report_file_path, output_file_path, server_name): + """Generate an enhanced interactive HTML report using templates.""" + specifications = parse_report_html(report_file_path) + + # Load the report template + template_dir = Path(__file__).parent / "templates" + template = load_template(template_dir / "report_template.html") + + # Create relative path to the duvet report.html + duvet_report_path = ".duvet/reports/report.html" + + # GitHub base URL - can be configured for when deployed to GitHub Pages + github_base_url = None + + # Calculate summary statistics + stats = calculate_summary_statistics(specifications) + + # Calculate percentages for each implementation type + total_reqs = stats["total_requirements"] + if total_reqs > 0: + # Calculate raw percentages + impl_test_pct = (stats["implementation_and_test"] / total_reqs) * 100 + impl_only_pct = (stats["implementation_only"] / total_reqs) * 100 + test_only_pct = (stats["test_only"] / total_reqs) * 100 + exception_pct = (stats["exception_count"] / total_reqs) * 100 + implication_pct = (stats["implication_count"] / total_reqs) * 100 + no_impl_pct = (stats["no_implementation"] / total_reqs) * 100 + + # Ensure percentages add up to exactly 100% by using precise calculation + # and assigning any remainder to the largest segment + if total_reqs > 0: + # Calculate exact percentages using integer arithmetic to avoid floating point errors + percentages_data = [ + (stats["implementation_and_test"], "impl_test"), + (stats["implication_count"], "implication"), + (stats["exception_count"], "exception"), + (stats["implementation_only"], "impl_only"), + (stats["no_implementation"], "no_impl"), + ] + + # Calculate percentages with high precision, then distribute remainder + total_allocated = 0.0 + calculated_percentages = {} + + # Calculate all but the last percentage + for i, (count, name) in enumerate(percentages_data[:-1]): + pct = round((count / total_reqs) * 100, 1) + calculated_percentages[name] = pct + total_allocated += pct + + # Last segment gets the remainder to ensure exactly 100% + last_count, last_name = percentages_data[-1] + calculated_percentages[last_name] = round(100.0 - total_allocated, 1) + + # Assign back to variables + impl_test_pct = calculated_percentages["impl_test"] + implication_pct = calculated_percentages["implication"] + exception_pct = calculated_percentages["exception"] + impl_only_pct = calculated_percentages["impl_only"] + no_impl_pct = calculated_percentages["no_impl"] + else: + impl_test_pct = impl_only_pct = test_only_pct = exception_pct = implication_pct = ( + no_impl_pct + ) = 0 + + # Generate summary statistics HTML with color-coded progress bars + content_html = f""" +
+
+
+
+ Requirements by Implementation Type + {stats['complete_requirements']}/{stats['total_requirements']} completed +
+
+
+
+
+
+
+
+
+
+ +
+
+
{stats['implementation_and_test']}
+
Implementation + Test
+
+
+
{stats['implication_count']}
+
Implication
+
+
+
{stats['exception_count']}
+
Exception
+
+
+
{stats['implementation_only']}
+
Implementation Only
+
+
+
{stats['no_implementation']}
+
No Implementation
+
+
+
{stats['total_requirements']}
+
Total
+
+
+
+ """ + + # Generate content for each specification + spec_counter = 0 + + for spec_title, spec_data in specifications.items(): + status_icon = get_spec_status(spec_data) + sections = spec_data.get("sections", {}) + + # Calculate requirement-level progress for this spec + spec_total_requirements = 0 + spec_complete_requirements = 0 + + for section_data in sections.values(): + section_requirements = section_data.get("requirements", []) + spec_total_requirements += len(section_requirements) + spec_complete_requirements += sum( + 1 for req in section_requirements if req["is_complete"] + ) + + # Determine alternating background class + row_class = "even" if spec_counter % 2 == 0 else "odd" + spec_counter += 1 + + # Generate spec-specific URL + spec_url = generate_spec_url(duvet_report_path, spec_data["spec_path"]) + + # Calculate spec-level statistics + spec_impl_test = 0 + spec_implication = 0 + spec_exception = 0 + spec_impl_only = 0 + spec_no_impl = 0 + + for section_data in sections.values(): + section_requirements = section_data.get("requirements", []) + for req in section_requirements: + if req["has_implementation"] and req["has_test"]: + spec_impl_test += 1 + elif req["has_implication"]: + spec_implication += 1 + elif req["has_exception"]: + spec_exception += 1 + elif req["has_implementation"]: + spec_impl_only += 1 + else: + spec_no_impl += 1 + + # Calculate percentages for spec progress bar + if spec_total_requirements > 0: + spec_impl_test_pct = (spec_impl_test / spec_total_requirements) * 100 + spec_implication_pct = (spec_implication / spec_total_requirements) * 100 + spec_exception_pct = (spec_exception / spec_total_requirements) * 100 + spec_impl_only_pct = (spec_impl_only / spec_total_requirements) * 100 + spec_no_impl_pct = (spec_no_impl / spec_total_requirements) * 100 + else: + spec_impl_test_pct = spec_implication_pct = spec_exception_pct = spec_impl_only_pct = ( + spec_no_impl_pct + ) = 0 + + content_html += f""" +
+
+
+ {status_icon} + {spec_title} + ({spec_complete_requirements}/{spec_total_requirements} completed) + 🔗 +
+ ▼ +
+ +
+""" + + # Add sections within each specification + for section_title, section_data in sections.items(): + section_requirements = section_data.get("requirements", []) + section_complete = sum(1 for req in section_requirements if req["is_complete"]) + section_total = len(section_requirements) + + # Skip sections with no requirements at all + if section_total == 0: + continue + + # Determine section status using the corrected logic + # Get individual requirement statuses + req_statuses = [get_requirement_status(req) for req in section_requirements] + + if all(status == "✅" for status in req_statuses): + section_status = "✅" # All requirements are green + elif any(status in ["✅", "🟡"] for status in req_statuses): + section_status = "🟡" # Any requirement is green or yellow + else: + section_status = "❌" # All requirements are red X + + section_id = f"{spec_title.replace(' ', '_')}_{section_title.replace(' ', '_').replace('#', '').replace('-', '_')}" + + # Generate section-specific URL + section_url = generate_section_url( + duvet_report_path, spec_data["spec_path"], section_data["section_id"] + ) + + # Generate local file path for this section + local_file_path = f"{spec_data['spec_path']}#{section_data['section_id']}" + + # Calculate section-level statistics + section_impl_test = sum( + 1 for req in section_requirements if req["has_implementation"] and req["has_test"] + ) + section_implication = sum(1 for req in section_requirements if req["has_implication"]) + section_exception = sum(1 for req in section_requirements if req["has_exception"]) + section_impl_only = sum( + 1 + for req in section_requirements + if req["has_implementation"] + and not req["has_test"] + and not req["has_exception"] + and not req["has_implication"] + ) + section_no_impl = sum( + 1 + for req in section_requirements + if not req["has_implementation"] + and not req["has_test"] + and not req["has_exception"] + and not req["has_implication"] + ) + + # Calculate percentages for section progress bar + if section_total > 0: + section_impl_test_pct = (section_impl_test / section_total) * 100 + section_implication_pct = (section_implication / section_total) * 100 + section_exception_pct = (section_exception / section_total) * 100 + section_impl_only_pct = (section_impl_only / section_total) * 100 + section_no_impl_pct = (section_no_impl / section_total) * 100 + else: + section_impl_test_pct = section_implication_pct = section_exception_pct = ( + section_impl_only_pct + ) = section_no_impl_pct = 0 + + content_html += f""" +
+
+
+ {section_status} + {section_title} + ({section_complete}/{section_total} completed) + 🔗 +
+ ▼ +
+
+ +
+ {local_file_path} + +
+""" + + # Add requirements within each section + req_counter = 1 + for requirement in section_requirements: + req_status = get_requirement_status(requirement) + req_text = format_requirement_text(requirement["text"]) + + # Build detailed source information with GitHub links - one bullet per source + sources_html = "" + if requirement["related_sources"]: + source_bullets = [] + for source_info in requirement["related_sources"]: + source_type = source_info["type"] + source_path = source_info["source"] + line_num = source_info["line"] + + # Generate GitHub URL if possible + github_url = generate_github_url(source_path, line_num, github_base_url) + + if github_url and source_path.endswith(".go"): + # Create clickable link for Go source files + source_display = f'{source_path}' + if line_num: + source_display += f":{line_num}" + source_display += "" + else: + # Plain text for non-Go files or when no GitHub URL + source_display = source_path + if line_num: + source_display += f":{line_num}" + + type_display = source_type.lower() + # Add partial indicator if this requirement has partial coverage + if requirement.get("has_partial_coverage", False): + type_display = f"partial {type_display}" + source_bullets.append(f"• {type_display}: {source_display}") + + sources_html = ( + '
' + + "
".join(source_bullets) + + "
" + ) + else: + sources_html = '
• no implementation found
' + + # Determine requirement type for filtering + if requirement["has_exception"]: + req_type = "exception" + elif requirement["has_implication"]: + req_type = "implication" + elif ( + requirement["has_implementation"] + and requirement["has_test"] + and not requirement.get("has_partial_coverage", False) + ): + req_type = "impl-test" + elif requirement["has_implementation"] and not requirement.get( + "has_partial_coverage", False + ): + req_type = "impl-only" + else: + # Partial coverage and no implementation both get "none" type + req_type = "none" + + # Prepare requirement text for copying (clean version without HTML) + clean_req_text = requirement["text"].replace("\n", " ").strip() + # Escape single quotes for JavaScript + clean_req_text = clean_req_text.replace("'", "\\'") + copy_text = f"//# {clean_req_text}" + + content_html += f""" +
+
+ Requirement {req_counter}: + {req_status} + +
+
{req_text}
+ {sources_html} +
+""" + req_counter += 1 + + content_html += """ +
+
+""" + + content_html += """ +
+
+""" + + # Replace placeholders in template + html_content = template.format(server_name=server_name, content=content_html) + + # Write the HTML file + with open(output_file_path, "w", encoding="utf-8") as f: + f.write(html_content) + + +def generate_server_report(server_path, server_name): + """Generate individual server report using the enhanced report-based format.""" + report_file = server_path / ".duvet" / "reports" / "report.html" + + if not report_file.exists(): + return None + + try: + # Parse the report directly + specifications = parse_report_html(report_file) + + # Generate the enhanced HTML report + html_output_file = server_path / "compliance_summary_report.html" + generate_enhanced_html_report(report_file, html_output_file, server_name) + + # Calculate detailed statistics + stats = calculate_summary_statistics(specifications) + + # Calculate overall status based on actual implementation progress + total_reqs = stats.get("total_requirements", 0) + complete_reqs = stats.get("complete_requirements", 0) + + if total_reqs == 0: + overall_status = "❌" # No requirements means not compliant + elif complete_reqs == total_reqs: + overall_status = "✅" # All requirements complete + elif complete_reqs > 0: + overall_status = "🟡" # Some requirements complete + else: + overall_status = "❌" # No requirements complete + + # Calculate spec-level status + spec_statuses = {} + for spec_title, spec_data in specifications.items(): + spec_statuses[spec_title] = get_spec_status(spec_data) + + total_specs = len(specifications) + complete_specs = sum(1 for status in spec_statuses.values() if status == "✅") + + return { + "name": server_name, + "status": overall_status, + "total_specs": total_specs, + "complete_specs": complete_specs, + "total_sections": stats["total_sections"], + "complete_sections": stats["complete_sections"], + "total_requirements": stats["total_requirements"], + "complete_requirements": stats["complete_requirements"], + "report_file": f"../{server_name}/compliance_summary_report.html", + "specifications": spec_statuses, + "stats": stats, # Include full stats for homepage display + } + + except Exception as e: + print(f"Error processing {server_name}: {e}") + return None + + +def generate_expected_output(report_file_path, output_file_path): + """Generate the expected output format from report.html.""" + specifications = parse_report_html(report_file_path) + + output_lines = [] + for spec_title, spec_data in specifications.items(): + status_icon = get_spec_status(spec_data) + output_lines.append(f"{spec_title}: {status_icon}") + + # Write the output file + with open(output_file_path, "w", encoding="utf-8") as f: + f.write("\n".join(output_lines)) + + +def generate_stats_output(report_file_path, output_file_path): + """Generate detailed statistics output for dashboard use.""" + specifications = parse_report_html(report_file_path) + stats = calculate_summary_statistics(specifications) + + # Write stats as JSON for easy parsing + import json + + with open(output_file_path, "w", encoding="utf-8") as f: + json.dump(stats, f, indent=2) + + +def generate_homepage(servers_info, output_file): + """Generate the main homepage with links to all server reports using templates.""" + + current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + # Load the homepage template + template_dir = Path(__file__).parent / "templates" + template = load_template(template_dir / "homepage_template.html") + + content_html = "" + + if servers_info: + # Calculate overall statistics + total_servers = len(servers_info) + compliant_servers = sum(1 for server in servers_info if server["status"] == "✅") + partial_servers = sum(1 for server in servers_info if server["status"] == "🟡") + non_compliant_servers = sum(1 for server in servers_info if server["status"] == "❌") + + # Add compact dark mode summary header + content_html += f""" +
+
+
+ {total_servers} +
Total
+
+
+ {compliant_servers} +
Compliant
+
+
+ {partial_servers} +
Partial
+
+
+ {non_compliant_servers} +
Missing
+
+
+
+ +
+""" + + # Generate server cards with detailed statistics + for server in sorted(servers_info, key=lambda x: x["name"]): + # Get detailed stats for this server + server_stats = server.get("stats", {}) + + # Calculate percentages for each implementation type + total_reqs = server_stats.get("total_requirements", 0) + if total_reqs > 0: + # Calculate raw percentages + impl_test_pct = (server_stats.get("implementation_and_test", 0) / total_reqs) * 100 + impl_only_pct = (server_stats.get("implementation_only", 0) / total_reqs) * 100 + test_only_pct = (server_stats.get("test_only", 0) / total_reqs) * 100 + exception_pct = (server_stats.get("exception_count", 0) / total_reqs) * 100 + implication_pct = (server_stats.get("implication_count", 0) / total_reqs) * 100 + no_impl_pct = (server_stats.get("no_implementation", 0) / total_reqs) * 100 + + # Ensure percentages add up to exactly 100% by using precise calculation + if total_reqs > 0: + # Calculate exact percentages and distribute remainder to largest segment + percentages_data = [ + (server_stats.get("implementation_and_test", 0), "impl_test"), + (server_stats.get("implication_count", 0), "implication"), + (server_stats.get("exception_count", 0), "exception"), + (server_stats.get("implementation_only", 0), "impl_only"), + (server_stats.get("no_implementation", 0), "no_impl"), + ] + + # Calculate percentages with high precision, then distribute remainder + total_allocated = 0.0 + calculated_percentages = {} + + # Calculate all but the last percentage + for i, (count, name) in enumerate(percentages_data[:-1]): + pct = round((count / total_reqs) * 100, 1) + calculated_percentages[name] = pct + total_allocated += pct + + # Last segment gets the remainder to ensure exactly 100% + last_count, last_name = percentages_data[-1] + calculated_percentages[last_name] = round(100.0 - total_allocated, 1) + + # Assign back to variables + impl_test_pct = calculated_percentages["impl_test"] + implication_pct = calculated_percentages["implication"] + exception_pct = calculated_percentages["exception"] + impl_only_pct = calculated_percentages["impl_only"] + no_impl_pct = calculated_percentages["no_impl"] + else: + impl_test_pct = impl_only_pct = test_only_pct = exception_pct = implication_pct = ( + no_impl_pct + ) = 0 + + content_html += f""" +
+
+
{server['name']}
+
{server['status']}
+
+
+
+
+ Requirements Progress + {server_stats.get('complete_requirements', 0)}/{server_stats.get('total_requirements', 0)} completed +
+
+
+
+
+
+
+
+
+ +
+
+
{server_stats.get('implementation_and_test', 0)}
+
Impl+Test
+
+
+
{server_stats.get('implication_count', 0)}
+
Implication
+
+
+
{server_stats.get('exception_count', 0)}
+
Exception
+
+
+
{server_stats.get('implementation_only', 0)}
+
Impl Only
+
+
+
{server_stats.get('no_implementation', 0)}
+
None
+
+
+
{server_stats.get('total_requirements', 0)}
+
Total
+
+
+
+
+ View Detailed Report +
+
+""" + + content_html += """ +
+""" + else: + content_html += """ +
+

No servers with compliance reports found.

+

Make sure servers have .duvet/reports/report.html files.

+
+""" + + # Replace placeholders in template + html_content = template.format(timestamp=current_time, content=content_html) + + # Write the HTML file + with open(output_file, "w", encoding="utf-8") as f: + f.write(html_content) + + +def discover_servers(): + """Discover all servers with .duvet/reports/report.html files.""" + servers_info = [] + # Get the test-server directory (parent of spec-compliance-dashboard) + test_server_dir = Path(__file__).parent.parent + + # Look for directories with .duvet/reports/report.html + for item in test_server_dir.iterdir(): + if ( + item.is_dir() + and not item.name.startswith(".") + and item.name != "spec-compliance-dashboard" + ): + duvet_report = item / ".duvet" / "reports" / "report.html" + if duvet_report.exists(): + server_info = generate_server_report(item, item.name) + if server_info: + servers_info.append(server_info) + print(f"Processed server: {item.name}") + + return servers_info + + +def main(): + """Main function to generate both individual server reports and dashboard.""" + import sys + + # Check if server directory is provided as argument (for single server mode) + if len(sys.argv) > 1: + server_dir = Path(sys.argv[1]) + server_name = sys.argv[2] if len(sys.argv) > 2 else server_dir.name + + report_file = server_dir / ".duvet" / "reports" / "report.html" + html_output_file = server_dir / "compliance_summary_report.html" + expected_output_file = server_dir / "expected_output_report.txt" + + if not report_file.exists(): + print(f"Error: Report file not found at {report_file}") + return 1 + + try: + # Generate HTML report + generate_enhanced_html_report(report_file, html_output_file, server_name) + print(f"Interactive HTML report generated: {html_output_file}") + + # Generate expected output + generate_expected_output(report_file, expected_output_file) + print(f"Expected output generated: {expected_output_file}") + + # Generate stats output for dashboard + stats_output_file = server_dir / "compliance_stats.json" + generate_stats_output(report_file, stats_output_file) + print(f"Stats output generated: {stats_output_file}") + + return 0 + except Exception as e: + print(f"Error generating reports: {e}") + return 1 + else: + # Dashboard mode - discover all servers and generate dashboard + try: + print("Discovering servers with compliance reports...") + servers_info = discover_servers() + + if servers_info: + print(f"Found {len(servers_info)} servers with reports") + + # Generate the main dashboard homepage + homepage_file = Path(__file__).parent / "compliance_homepage.html" + generate_homepage(servers_info, homepage_file) + print(f"Dashboard homepage generated: {homepage_file}") + + return 0 + else: + print("No servers with .duvet/reports/report.html found") + return 1 + + except Exception as e: + print(f"Error generating dashboard: {e}") + return 1 + + +if __name__ == "__main__": + exit(main()) diff --git a/test-server/spec-compliance-dashboard/templates/homepage_styles.css b/test-server/spec-compliance-dashboard/templates/homepage_styles.css new file mode 100644 index 00000000..466f1393 --- /dev/null +++ b/test-server/spec-compliance-dashboard/templates/homepage_styles.css @@ -0,0 +1,335 @@ +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif; + line-height: 1.6; + margin: 0; + padding: 20px; + background-color: #0d1117; + color: #c9d1d9; +} + +.container { + max-width: 1200px; + margin: 0 auto; + background: #161b22; + border-radius: 6px; + box-shadow: 0 1px 3px rgba(0,0,0,0.3); + border: 1px solid #30363d; + overflow: hidden; +} + +.header { + background: #21262d; + color: #c9d1d9; + padding: 15px 20px; + text-align: center; + border-bottom: 1px solid #30363d; +} + +.header h1 { + margin: 0; + font-size: 1.8em; + font-weight: 400; +} + +.header p { + margin: 5px 0 0 0; + opacity: 0.9; + font-size: 0.9em; +} + +.stats-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: 20px; + padding: 30px; + background: #0d1117; +} + +.stat-card { + background: #161b22; + padding: 20px; + border-radius: 6px; + text-align: center; + box-shadow: 0 1px 3px rgba(0,0,0,0.3); + border: 1px solid #30363d; +} + +.stat-number { + font-size: 2em; + font-weight: bold; + color: #c9d1d9; +} + +.stat-label { + color: #8b949e; + font-size: 0.9em; + margin-top: 5px; +} + +.servers-grid { + display: grid; + grid-template-columns: repeat(3, 1fr); + grid-template-rows: repeat(2, 1fr); + gap: 20px; + padding: 30px; + max-width: 1200px; + margin: 0 auto; +} + +.server-card { + background: #161b22; + border-radius: 6px; + box-shadow: 0 1px 3px rgba(0,0,0,0.3); + border: 1px solid #30363d; + overflow: hidden; + transition: transform 0.2s, box-shadow 0.2s; +} + +.server-card:hover { + transform: translateY(-2px); + box-shadow: 0 3px 6px rgba(0,0,0,0.4); +} + +.server-header { + padding: 12px 16px; + background: #21262d; + color: #c9d1d9; + display: flex; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid #30363d; +} + +.server-name { + font-size: 1.2em; + font-weight: 600; +} + +.server-status { + font-size: 1.5em; +} + +.server-body { + padding: 20px; +} + +.progress-bar { + background: #0d1117; + border-radius: 6px; + height: 8px; + margin: 15px 0; + overflow: hidden; + border: 1px solid #30363d; +} + +.progress-fill { + height: 100%; + background: #238636; + border-radius: 6px; + transition: width 0.3s ease; +} + +.progress-bar.color-coded { + display: flex; + height: 12px; +} + +.progress-segment { + height: 100%; + transition: width 0.3s ease; + position: relative; +} + +.progress-segment:first-child { + border-radius: 6px 0 0 6px; +} + +.progress-segment:last-child { + border-radius: 0 6px 6px 0; +} + +.progress-segment:only-child { + border-radius: 6px; +} + +.progress-segment.impl-test { + background: #28a745; +} + +.progress-segment.impl-only { + background: #ffc107; +} + +.progress-segment.test-only { + background: #6c757d; +} + +.progress-segment.exception { + background: #87ceeb; +} + +.progress-segment.implication { + background: #dda0dd; +} + +.progress-segment.no-impl { + background: #dc3545; +} + +.progress-item { + margin-bottom: 15px; +} + +.progress-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 8px; +} + +.progress-label { + color: #c9d1d9; + font-weight: 500; + font-size: 0.9em; +} + +.progress-count { + color: #8b949e; + font-size: 0.8em; +} + +.breakdown-grid-compact { + display: grid; + grid-template-columns: repeat(5, 1fr); + gap: 8px; + margin-top: 10px; +} + +.breakdown-item-compact { + background: #0d1117; + padding: 8px 4px; + border-radius: 4px; + text-align: center; + border: 1px solid #30363d; +} + +.breakdown-number-compact { + font-size: 1.1em; + font-weight: bold; + color: #c9d1d9; + margin-bottom: 2px; +} + +.breakdown-label-compact { + color: #8b949e; + font-size: 0.7em; + line-height: 1.2; +} + +/* Regular breakdown grid (used by the generated HTML) */ +.breakdown-grid { + display: grid; + grid-template-columns: repeat(3, 1fr); + grid-template-rows: repeat(2, 1fr); + gap: 8px; + margin-top: 10px; +} + +.breakdown-item { + background: transparent; + padding: 8px 4px; + border-radius: 4px; + text-align: center; + border: none; +} + +.breakdown-number { + font-size: 1.1em; + font-weight: bold; + color: #c9d1d9; + margin-bottom: 2px; +} + +.breakdown-label { + color: #8b949e; + font-size: 0.7em; + line-height: 1.2; +} + +.server-summary { + margin-top: 15px; +} + +.summary-row { + display: flex; + justify-content: space-between; + margin-bottom: 8px; +} + +.summary-row:last-child { + margin-bottom: 0; +} + +.summary-item { + display: flex; + flex-direction: column; + align-items: center; + flex: 1; +} + +.summary-number { + font-size: 1.2em; + font-weight: bold; + margin-bottom: 2px; +} + +.summary-label { + color: #8b949e; + font-size: 0.75em; + text-align: center; +} + +.server-stats { + display: flex; + justify-content: space-between; + margin-top: 15px; + font-size: 0.9em; + color: #8b949e; +} + +.server-footer { + padding: 15px 20px; + background: #0d1117; + border-top: 1px solid #30363d; + text-align: center; +} + +.view-report-btn { + display: inline-block; + padding: 10px 20px; + background: #238636; + color: white; + text-decoration: none; + border-radius: 6px; + transition: background-color 0.2s; + font-size: 0.9em; +} + +.view-report-btn:hover { + background: #2ea043; +} + +.no-data { + text-align: center; + padding: 40px; + color: #8b949e; +} + +.footer { + padding: 20px; + text-align: center; + background: #21262d; + color: #8b949e; + font-size: 0.9em; + border-top: 1px solid #30363d; +} diff --git a/test-server/spec-compliance-dashboard/templates/homepage_template.html b/test-server/spec-compliance-dashboard/templates/homepage_template.html new file mode 100644 index 00000000..eddc8a4d --- /dev/null +++ b/test-server/spec-compliance-dashboard/templates/homepage_template.html @@ -0,0 +1,21 @@ + + + + + + Spec Compliance Dashboard + + + +
+
+

Spec Compliance Dashboard

+

Last updated: {timestamp}

+
+ {content} +
+ Generated by duvet compliance dashboard • {timestamp} +
+
+ + diff --git a/test-server/spec-compliance-dashboard/templates/report_template.html b/test-server/spec-compliance-dashboard/templates/report_template.html new file mode 100644 index 00000000..94d06ff8 --- /dev/null +++ b/test-server/spec-compliance-dashboard/templates/report_template.html @@ -0,0 +1,276 @@ + + + + + + {server_name} - Duvet Compliance Report + + + +
+
+

{server_name} + + 📊 Duvet Report + +

+ Back to Homepage +
+ {content} +
+ + + + diff --git a/test-server/spec-compliance-dashboard/templates/styles.css b/test-server/spec-compliance-dashboard/templates/styles.css new file mode 100644 index 00000000..161175d6 --- /dev/null +++ b/test-server/spec-compliance-dashboard/templates/styles.css @@ -0,0 +1,387 @@ +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif; + line-height: 1.6; + margin: 0; + padding: 20px; + background-color: #0d1117; + color: #c9d1d9; +} + +.container { + max-width: 1000px; + margin: 0 auto; + background: #161b22; + border-radius: 6px; + box-shadow: 0 1px 3px rgba(0,0,0,0.3); + border: 1px solid #30363d; + overflow: hidden; +} + +.header { + background: #21262d; + color: #c9d1d9; + padding: 8px 15px; + display: flex; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid #30363d; +} + +.header h1 { + margin: 0; + font-size: 1.2em; + font-weight: 500; +} + +.nav-link { + color: white; + text-decoration: none; + font-size: 0.9em; + opacity: 0.9; +} + +.nav-link:hover { + opacity: 1; + text-decoration: underline; +} + +.spec-section { + border-bottom: 1px solid #30363d; +} + +.spec-section.even { + background: #161b22; +} + +.spec-section.odd { + background: #0d1117; +} + +.spec-header { + padding: 15px 20px; + cursor: pointer; + display: flex; + justify-content: space-between; + align-items: center; + background: transparent; + transition: background-color 0.2s; + color: #c9d1d9; +} + +.spec-header:hover { + background: #21262d; +} + +.spec-title { + font-size: 18px; + font-weight: 600; + display: flex; + align-items: center; + gap: 10px; +} + +.completion-count { + color: #8b949e; + font-size: 0.8em; + font-weight: 400; +} + +.status-emoji { + font-size: 20px; +} + +.expand-icon { + font-size: 14px; + transition: transform 0.2s; +} + +.spec-content { + display: none; + padding: 20px; + background: transparent; +} + +.spec-content.expanded { + display: block; +} + +.requirement-item { + margin-bottom: 15px; + padding: 15px; + background: #161b22; + border-radius: 6px; + box-shadow: 0 1px 3px rgba(0,0,0,0.3); + border: 1px solid #30363d; + color: #c9d1d9; +} + +.requirement-header { + display: flex; + align-items: center; + gap: 10px; + margin-bottom: 10px; + color: #c9d1d9; +} + +.requirement-id { + font-weight: bold; + color: #c9d1d9; +} + +.requirement-status { + font-size: 16px; +} + +.requirement-text { + color: #c9d1d9; + white-space: pre-wrap; + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace; + font-size: 14px; + line-height: 1.4; +} + +.section-item { + margin-bottom: 10px; + border-radius: 6px; + background: #21262d; + border: 1px solid #30363d; +} + +.section-header { + padding: 12px 15px; + cursor: pointer; + display: flex; + justify-content: space-between; + align-items: center; + background: transparent; + transition: background-color 0.2s; + color: #c9d1d9; +} + +.section-header:hover { + background: #30363d; +} + +.section-title { + font-size: 16px; + font-weight: 500; + display: flex; + align-items: center; + gap: 8px; +} + +.section-content { + display: none; + padding: 15px; + background: transparent; +} + +.section-content.expanded { + display: block; +} + +.requirement-metadata { + color: #8b949e; + font-size: 12px; + margin-top: 8px; + font-style: italic; +} + +.status-metadata { + color: #6e7681; + font-size: 12px; + font-style: italic; +} + +.summary-stats { + padding: 20px; + background: #0d1117; + border-bottom: 1px solid #30363d; +} + +.summary-stats h2 { + margin: 0 0 15px 0; + color: #c9d1d9; + font-size: 1.4em; + font-weight: 600; +} + +.summary-stats h3 { + margin: 20px 0 10px 0; + color: #c9d1d9; + font-size: 1.1em; + font-weight: 500; +} + +.progress-section { + margin-bottom: 20px; +} + +.progress-item { + margin-bottom: 15px; +} + +.progress-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 8px; +} + +.progress-label { + color: #c9d1d9; + font-weight: 500; +} + +.progress-count { + color: #8b949e; + font-size: 0.9em; +} + +.progress-bar { + background: #21262d; + border-radius: 6px; + height: 8px; + overflow: hidden; + border: 1px solid #30363d; +} + +.progress-fill { + height: 100%; + background: #238636; + border-radius: 6px; + transition: width 0.3s ease; +} + +.progress-bar.color-coded { + display: flex; + height: 12px; +} + +.progress-segment { + height: 100%; + transition: width 0.3s ease; + position: relative; +} + +.progress-segment:first-child { + border-radius: 6px 0 0 6px; +} + +.progress-segment:last-child { + border-radius: 0 6px 6px 0; +} + +.progress-segment:only-child { + border-radius: 6px; +} + +.progress-segment.impl-test { + background: #28a745; +} + +.progress-segment.impl-only { + background: #ffc107; +} + +.progress-segment.test-only { + background: #6c757d; +} + +.progress-segment.exception { + background: #87ceeb; +} + +.progress-segment.implication { + background: #dda0dd; +} + +.progress-segment.no-impl { + background: #dc3545; +} + +.breakdown-grid { + display: grid; + grid-template-columns: repeat(3, 1fr); + grid-template-rows: repeat(2, 1fr); + gap: 10px; +} + +.breakdown-grid.single-row { + grid-template-columns: repeat(6, 1fr); + grid-template-rows: 1fr; +} + +.breakdown-item { + background: transparent; + padding: 12px; + border-radius: 6px; + text-align: center; + border: none; + transition: all 0.2s ease; +} + +.breakdown-item.clickable-filter { + cursor: pointer; + border: 1px solid transparent; +} + +.breakdown-item.clickable-filter:hover { + background: #21262d; + border: 1px solid #30363d; + transform: translateY(-1px); +} + +.breakdown-item.active-filter { + background: #21262d; + border: 2px solid #58a6ff; + box-shadow: 0 0 8px rgba(88, 166, 255, 0.3); +} + +.breakdown-number { + font-size: 1.4em; + font-weight: bold; + color: #c9d1d9; + margin-bottom: 3px; +} + +.breakdown-label { + color: #8b949e; + font-size: 0.8em; +} + +.breakdown-header { + display: flex; + justify-content: space-between; + align-items: center; + cursor: pointer; + padding: 10px 0; + border-bottom: 1px solid #30363d; + margin-bottom: 15px; +} + +.breakdown-header:hover { + background: #21262d; + border-radius: 6px; + padding: 10px 15px; + margin: 0 -15px 15px -15px; +} + +.breakdown-header h3 { + margin: 0; +} + +.pie-chart-container { + background: #161b22; + padding: 20px; + border-radius: 6px; + border: 1px solid #30363d; + margin-top: 15px; + justify-content: center; + align-items: center; +} + +.pie-chart-container canvas { + max-width: 100%; + height: auto; +} diff --git a/test-server/spec-compliance-dashboard/templates/summary_stats_template.html b/test-server/spec-compliance-dashboard/templates/summary_stats_template.html new file mode 100644 index 00000000..0415d138 --- /dev/null +++ b/test-server/spec-compliance-dashboard/templates/summary_stats_template.html @@ -0,0 +1,63 @@ +
+

Summary Statistics

+
+
+
+ Sections Implemented + {complete_sections}/{total_sections} +
+
+
+
+
+
+
+ Requirements Implemented + {complete_requirements}/{total_requirements} +
+
+
+
+
+
+ +
+

Implementation Breakdown

+ ▼ +
+
+
+
{implementation_and_test}
+
Implementation + Test
+
+
+
{implementation_only}
+
Implementation Only
+
+
+
{test_only}
+
Test Only
+
+
+
{exception_count}
+
Exception
+
+
+
{implication_count}
+
Implication
+
+
+
{no_implementation}
+
No Implementation
+
+
+ + + + +
diff --git a/test-server/specification b/test-server/specification new file mode 160000 index 00000000..1f1ae8bb --- /dev/null +++ b/test-server/specification @@ -0,0 +1 @@ +Subproject commit 1f1ae8bb2b7b082b87ffbf4916a9723e531b2052 From 4c9a1e9f6da832eb5e375d7441e9edfebcc36534 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Tue, 17 Feb 2026 09:35:04 -0800 Subject: [PATCH 55/81] chore: refactor to make S3EC install itself as plugins (#138) * add multithreaded EncCtx integ tests --- Makefile | 4 +- pyproject.toml | 1 + src/s3_encryption/__init__.py | 223 ++++++++----- src/s3_encryption/materials/kms_keyring.py | 2 +- src/s3_encryption/pipelines.py | 5 +- test/integration/test_i_s3_encryption.py | 197 +++++++++++- .../test_i_s3_encryption_multithreaded.py | 303 ++++++++++++++++++ test/test_decryption_materials_integration.py | 39 ++- 8 files changed, 661 insertions(+), 113 deletions(-) create mode 100644 test/integration/test_i_s3_encryption_multithreaded.py diff --git a/Makefile b/Makefile index 814ab334..7db980c3 100644 --- a/Makefile +++ b/Makefile @@ -10,14 +10,14 @@ install: # Run linting checks lint: - uv run black --check . + uv run black --check src/ test/ # Enforce ruff checks on src/ but allow test/ to fail uv run ruff check src/ uv run ruff check test/ || true # Format code with Black and Ruff format: - uv run black . + uv run black src/ test/ uv run ruff check --fix src/ test/ # Run all tests diff --git a/pyproject.toml b/pyproject.toml index b05f22aa..b7489a03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ exclude = [".git", "__pycache__", "build", "dist"] [tool.ruff.lint] # Enable all rules by default, then configure specific rule settings below select = ["E", "F", "W", "I", "N", "D", "UP", "B", "A", "C4", "PT", "RET", "SIM", "ARG", "ERA"] +ignore = ["ARG002"] # Allow unused method arguments (e.g., **kwargs for API compatibility) [tool.ruff.lint.pydocstyle] convention = "google" diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 46cdbdd1..064096bb 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -3,9 +3,9 @@ """Top-level S3 Encryption Client v3 for Python package.""" import io +import threading from attrs import define, field -from botocore import serialize from botocore.response import StreamingBody from .exceptions import S3EncryptionClientError @@ -16,7 +16,7 @@ from .materials.keyring import AbstractKeyring from .pipelines import GetEncryptedObjectPipeline, PutEncryptedObjectPipeline -DEFAULT_ENCODING = "utf-8" +S3_METADATA_PREFIX = "x-amz-meta-" @define @@ -31,31 +31,122 @@ def _default_cmm_for_keyring(self): return DefaultCryptoMaterialsManager(self.keyring) +class S3EncryptionClientPlugin: + """Plugin that adds encryption/decryption capabilities to a boto3 S3 client. + + This plugin uses boto3's event system to intercept put_object and get_object + calls to provide transparent encryption and decryption of S3 objects. + """ + + def __init__(self, config: S3EncryptionClientConfig): + """Initialize the plugin with encryption configuration. + + Args: + config: S3EncryptionClientConfig containing keyring and CMM + """ + self.config = config + self._context = threading.local() + + def on_put_object_before_call(self, params, **kwargs): + """Event handler for before-call.s3.PutObject. + + This handler encrypts the body after serialization but before the request is sent. + + Args: + params: Dictionary of parameters for the PutObject call (after serialization) + **kwargs: Additional event arguments + """ + # At this point, boto3 has already serialized the Body + # Extract the serialized body from the request + body = params.get("body") + if body is None: + body_bytes = b"" + elif isinstance(body, bytes): + body_bytes = body + elif hasattr(body, "read"): + # It's a file-like object (BytesIO, etc.) + # TODO(streaming): Add support for streaming encryption without reading entire body + # into memory + body_bytes = body.read() + else: + # Unexpected body type - should not happen as boto3 validates before this point + raise S3EncryptionClientError("Unexpected type of body parameter!") + + encryption_context = getattr(self._context, "encryption_context", None) + + pipeline = PutEncryptedObjectPipeline(self.config.cmm) + encrypted_data, encryption_metadata = pipeline.encrypt( + body_bytes, encryption_context=encryption_context + ) + + params["body"] = encrypted_data + + headers = params.get("headers", {}) + + # Add encryption metadata to headers + if encryption_metadata: + for key, value in encryption_metadata.items(): + # Add as S3 metadata headers + header_key = f"{S3_METADATA_PREFIX}{key}" + headers[header_key] = value + + params["headers"] = headers + + def on_get_object_after_call(self, parsed, **kwargs): + """Event handler for after-call.s3.GetObject. + + This handler decrypts the body after the response is received from S3. + + Args: + parsed: Dictionary containing the parsed response + **kwargs: Additional event arguments (includes 'params' with request parameters) + """ + # Get encryption context from thread-local storage (set by get_object wrapper) + encryption_context = getattr(self._context, "encryption_context", None) + + # The parsed response already has the Body as a StreamingBody + # We need to read it, decrypt it, and replace it + + # Create a response dict that matches what the pipeline expects + response = { + "Body": parsed.get("Body"), + "Metadata": parsed.get("Metadata", {}), + } + + # Create a pipeline and decrypt the data + pipeline = GetEncryptedObjectPipeline(self.config.cmm) + decrypted_data = pipeline.decrypt(response, encryption_context) + + # Replace body with decrypted data + stream = io.BytesIO(decrypted_data) + streaming_body = StreamingBody(stream, len(decrypted_data)) + parsed["Body"] = streaming_body + + @define class S3EncryptionClient: """Client for encrypting and decrypting S3 objects. This client wraps a boto3 S3 client and provides encryption and decryption capabilities for S3 objects using the configured keyring and crypto materials manager. + + The encryption/decryption is implemented using boto3's event system, registering + handlers for before-call and after-call events. """ wrapped_s3_client = field() config: S3EncryptionClientConfig = field() + _plugin: S3EncryptionClientPlugin = field(init=False) def __attrs_post_init__(self): - """Validate serialization encoding after initialization. + """Install the encryption plugin on the wrapped client using boto3 events.""" + # Create the plugin + object.__setattr__(self, "_plugin", S3EncryptionClientPlugin(self.config)) - Ensures boto3 serializers are using the expected default encoding. - """ - # Sanity check that boto3 serialization are ONLY using the default encoding (utf-8) - # This should always be the case, but changes in encoding would break the assumption that - # the decrypted plaintext adheres to the non-utf8 encoding scheme. So we avoid that. - for sz_name, sz in serialize.SERIALIZERS.items(): - if sz.DEFAULT_ENCODING != DEFAULT_ENCODING: - raise S3EncryptionClientError( - f"All Serializers MUST only support utf-8 encoding, but {sz_name} is using " - f"{sz.DEFAULT_ENCODING}!" - ) + # Register event handlers using boto3's event system + event_system = self.wrapped_s3_client.meta.events + event_system.register("before-call.s3.PutObject", self._plugin.on_put_object_before_call) + event_system.register("after-call.s3.GetObject", self._plugin.on_get_object_after_call) def put_object(self, **kwargs): """Encrypt and upload an object to S3. @@ -71,52 +162,28 @@ def put_object(self, **kwargs): Returns: The response from the S3 client's put_object method. + + Raises: + S3EncryptionClientError: Any problem with encryption, including if the Body parameter + has an invalid type. """ - # Extract required parameters from kwargs - bucket = kwargs.pop("Bucket") - key = kwargs.pop("Key") - body = kwargs.pop("Body", b"") # Default to empty bytes when Body is not provided + # Extract EncryptionContext if provided (not a standard S3 parameter) encryption_context = kwargs.pop("EncryptionContext", None) - # Create a pipeline for this operation - pipeline = PutEncryptedObjectPipeline(self.config.cmm) - - # The documentation for boto3 asks for bytes or a file-like object, - # but in reality, it is possible to pass strings. - # Strings will be encoded using DEFAULT_ENCODING, - # which MUST match the default encoding defined int the Serializer class in botocore. - if isinstance(body, str): - data_bytes = body.encode(DEFAULT_ENCODING) - elif isinstance(body, bytes): - data_bytes = body - elif isinstance(body, io.IOBase): - # TODO: Streaming support - raise S3EncryptionClientError( - f"Body parameter of type {type(body)} is not an acceptable type! " - f"Streaming operations are not yet supported." - ) - else: - raise S3EncryptionClientError( - f"Body parameter of type {type(body)} is not an acceptable type! " - f"Use bytes or a file-like object." - ) - - # Now encrypt the bytes/file-like IOBase object - encrypted_data, encryption_metadata = pipeline.encrypt( - data_bytes, encryption_context=encryption_context - ) - - # Add encryption metadata to the request parameters - params = {"Bucket": bucket, "Key": key, "Body": encrypted_data, **kwargs} - - # Add encryption metadata to the parameters - if encryption_metadata: - # Merge any existing metadata with our encryption metadata - metadata = params.get("Metadata", {}) - metadata.update(encryption_metadata) - params["Metadata"] = metadata - - return self.wrapped_s3_client.put_object(**params) + # Store encryption context in thread-local storage for the event handler + self._plugin._context.encryption_context = encryption_context + + try: + return self.wrapped_s3_client.put_object(**kwargs) + except S3EncryptionClientError: + # Re-raise our own exceptions without wrapping + raise + except Exception as e: + raise S3EncryptionClientError(f"Failed to encrypt object: {str(e)}") from e + finally: + # Clean up thread-local storage + if hasattr(self._plugin._context, "encryption_context"): + delattr(self._plugin._context, "encryption_context") def get_object(self, **kwargs): """Download and decrypt an object from S3. @@ -131,29 +198,25 @@ def get_object(self, **kwargs): Returns: The response from the S3 client's get_object method with the Body replaced with a StreamingBody containing the decrypted data. + + Raises: + S3EncryptionClientError: If decryption fails or the object is not properly encrypted. """ - # Extract encryption context if provided + # Extract EncryptionContext if provided (not a standard S3 parameter) encryption_context = kwargs.pop("EncryptionContext", None) - # Create params for the S3 client - params = {**kwargs} - - # Get the encrypted object from S3 - response = self.wrapped_s3_client.get_object(**params) - - # Create a pipeline for this operation - pipeline = GetEncryptedObjectPipeline(self.config.cmm) - - # Decrypt the data using the pipeline - decrypted_data = pipeline.decrypt( - response, encryption_context - ) # encrypted_data, encryption_metadata) - - # Create a new streaming body with the decrypted data - stream = io.BytesIO(decrypted_data) - streaming_body = StreamingBody(stream, len(decrypted_data)) - - # Update the response with the decrypted data - response["Body"] = streaming_body - - return response + # Store encryption context in thread-local storage for the event handler + self._plugin._context.encryption_context = encryption_context + + try: + return self.wrapped_s3_client.get_object(**kwargs) + except S3EncryptionClientError: + # Re-raise our own exceptions without wrapping + raise + except Exception as e: + # Wrap any unexpected errors during decryption + raise S3EncryptionClientError(f"Failed to decrypt object: {str(e)}") from e + finally: + # Clean up thread-local storage + if hasattr(self._plugin._context, "encryption_context"): + delattr(self._plugin._context, "encryption_context") diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index 7bc8f7bd..f8bc4997 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -46,7 +46,6 @@ def on_encrypt(self, enc_materials): # Call parent class validation enc_materials = super().on_encrypt(enc_materials) - # Add default encryption context encryption_context = enc_materials.encryption_context encryption_context["aws:x-amz-cek-alg"] = "AES/GCM/NoPadding" @@ -111,6 +110,7 @@ def on_decrypt(self, dec_materials, encrypted_data_keys=None): encryption_context_stored_copy = encryption_context_stored.copy() encryption_context_stored_copy.pop(KMS_V1_DEFAULT_KEY, None) encryption_context_stored_copy.pop(KMS_CONTEXT_DEFAULT_KEY, None) + if encryption_context_stored_copy != encryption_context_from_request: # TODO: modeled error raise S3EncryptionClientError( diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 37093803..6867ed7c 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -39,9 +39,9 @@ def encrypt(self, plaintext, encryption_context=None): bytes: The encrypted data dict: Metadata about the encryption to be stored with the object """ - # Create encryption materials request with encryption context + # Create encryption materials request with encryption context copy enc_mats_request = EncryptionMaterials( - encryption_context={} if encryption_context is None else encryption_context + encryption_context={} if encryption_context is None else encryption_context.copy() ) # Get encryption materials from the crypto materials manager @@ -102,6 +102,7 @@ def decrypt(self, response, encryption_context=None): bytes: The decrypted data """ # Convert the metadata dictionary to an ObjectMetadata instance + # TODO: Stream + Buffered Decryption encrypted_data = response.get("Body").read() encryption_metadata = response.get("Metadata", {}) metadata = ObjectMetadata.from_dict(encryption_metadata) diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index 2c8ea73a..616f8da4 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -216,7 +216,7 @@ def test_binary_data_roundtrip(): key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") # Create some binary data (not valid in any particular encoding) - data = bytes([i for i in range(256)]) + data = bytes(range(256)) kms_client = boto3.client("kms", region_name=region) @@ -259,31 +259,214 @@ def test_invalid_body_types(): # Test with integer with pytest.raises(S3EncryptionClientError) as excinfo: s3ec.put_object(Bucket=bucket, Key=key, Body=42) - assert "not an acceptable type" in str(excinfo.value) + assert "Invalid type for parameter Body" in str(excinfo.value) # Test with float with pytest.raises(S3EncryptionClientError) as excinfo: s3ec.put_object(Bucket=bucket, Key=key, Body=3.14) - assert "not an acceptable type" in str(excinfo.value) + assert "Invalid type for parameter Body" in str(excinfo.value) # Test with list with pytest.raises(S3EncryptionClientError) as excinfo: s3ec.put_object(Bucket=bucket, Key=key, Body=[1, 2, 3]) - assert "not an acceptable type" in str(excinfo.value) + assert "Invalid type for parameter Body" in str(excinfo.value) # Test with dictionary with pytest.raises(S3EncryptionClientError) as excinfo: s3ec.put_object(Bucket=bucket, Key=key, Body={"key": "value"}) - assert "not an acceptable type" in str(excinfo.value) + assert "Invalid type for parameter Body" in str(excinfo.value) # Test with boolean with pytest.raises(S3EncryptionClientError) as excinfo: s3ec.put_object(Bucket=bucket, Key=key, Body=True) - assert "not an acceptable type" in str(excinfo.value) + assert "Invalid type for parameter Body" in str(excinfo.value) # Test with None (also raises an exception) with pytest.raises(S3EncryptionClientError) as excinfo: s3ec.put_object(Bucket=bucket, Key=key, Body=None) - assert "not an acceptable type" in str(excinfo.value) + assert "Invalid type for parameter Body" in str(excinfo.value) print("Success! All invalid body types correctly raised exceptions.") + + +def test_user_metadata_preservation(): + """Test that user-provided metadata is preserved during encryption.""" + key = "metadata-preservation-rt" + key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + + data = "Test data with user metadata" + + # User metadata to include + user_metadata = { + "author": "test-user", + "version": "1.0", + "description": "Test object with custom metadata", + } + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + # Put object with user metadata + s3ec.put_object(Bucket=bucket, Key=key, Body=data, Metadata=user_metadata) + + # Get the object back + get_req = {"Bucket": bucket, "Key": key} + response = s3ec.get_object(**get_req) + + # Verify the data decrypts correctly + output = response["Body"].read().decode("utf-8") + if output != data: + print("Uh oh! Input and output don't match!") + print("Input:") + print(repr(data)) + print("Output:") + print(repr(output)) + raise RuntimeError + + # Verify user metadata is preserved + returned_metadata = response.get("Metadata", {}) + + for key_name, expected_value in user_metadata.items(): + if key_name not in returned_metadata: + print(f"Uh oh! User metadata key '{key_name}' is missing!") + print("Expected metadata:") + print(user_metadata) + print("Returned metadata:") + print(returned_metadata) + raise RuntimeError + + if returned_metadata[key_name] != expected_value: + print(f"Uh oh! User metadata value for '{key_name}' doesn't match!") + print(f"Expected: {expected_value}") + print(f"Got: {returned_metadata[key_name]}") + raise RuntimeError + + print("Success! User metadata preserved correctly during encryption/decryption.") + print(f"User metadata: {user_metadata}") + print(f"Returned metadata keys: {list(returned_metadata.keys())}") + + +def test_encryption_context_roundtrip(): + """Test that EncryptionContext is properly used during encryption and required for decryption.""" + key = "encryption-context-rt" + key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + + data = "Test data with encryption context" + + # Encryption context to use for additional authenticated data + encryption_context = { + "department": "engineering", + "project": "s3-encryption", + "environment": "test", + } + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + # Put object with encryption context + s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) + + # Get the object back WITH the same encryption context + get_req = {"Bucket": bucket, "Key": key, "EncryptionContext": encryption_context} + response = s3ec.get_object(**get_req) + + # Verify the data decrypts correctly + output = response["Body"].read().decode("utf-8") + if output != data: + print("Uh oh! Input and output don't match!") + print("Input:") + print(repr(data)) + print("Output:") + print(repr(output)) + raise RuntimeError + + print("Success! Encryption context used correctly during encryption/decryption.") + print(f"Encryption context: {encryption_context}") + + +def test_encryption_context_mismatch(): + """Test that decryption fails when EncryptionContext doesn't match.""" + key = "encryption-context-mismatch" + key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + + data = "Test data with encryption context" + + # Original encryption context + encryption_context = {"department": "engineering", "project": "s3-encryption"} + + # Wrong encryption context for decryption + wrong_encryption_context = {"department": "marketing", "project": "s3-encryption"} + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + # Put object with encryption context + s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) + + # Try to get the object back with WRONG encryption context - should fail + get_req = {"Bucket": bucket, "Key": key, "EncryptionContext": wrong_encryption_context} + + try: + s3ec.get_object(**get_req) + # If we get here, the test failed - decryption should have failed + print("Uh oh! Decryption succeeded with wrong encryption context!") + print(f"Original context: {encryption_context}") + print(f"Wrong context used: {wrong_encryption_context}") + raise RuntimeError("Expected decryption to fail with mismatched encryption context") + except S3EncryptionClientError as e: + # This is expected - decryption should fail + print("Success! Decryption correctly failed with mismatched encryption context.") + print(f"Error message: {str(e)}") + except Exception as e: + # Some other error occurred + print(f"Unexpected error type: {type(e).__name__}") + print(f"Error message: {str(e)}") + raise + + +def test_encryption_context_missing_on_decrypt(): + """Test that decryption fails when encryption context is not provided for an object encrypted with context.""" + key = "encryption-context-missing" + key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + + data = "Test data with encryption context" + + # Encryption context used during encryption + encryption_context = {"department": "engineering", "project": "s3-encryption"} + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + # Put object with encryption context + s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) + + # Try to get the object back WITHOUT encryption context - should fail + get_req = {"Bucket": bucket, "Key": key} + + try: + s3ec.get_object(**get_req) + # If we get here, the test failed - decryption should have failed + print("Uh oh! Decryption succeeded without providing required encryption context!") + print(f"Original context: {encryption_context}") + raise RuntimeError("Expected decryption to fail when encryption context not provided") + except S3EncryptionClientError as e: + # This is expected - decryption should fail + print("Success! Decryption correctly failed when encryption context was not provided.") + print(f"Error message: {str(e)}") + except Exception as e: + # Some other error occurred + print(f"Unexpected error type: {type(e).__name__}") + print(f"Error message: {str(e)}") + raise diff --git a/test/integration/test_i_s3_encryption_multithreaded.py b/test/integration/test_i_s3_encryption_multithreaded.py new file mode 100644 index 00000000..419ca7ea --- /dev/null +++ b/test/integration/test_i_s3_encryption_multithreaded.py @@ -0,0 +1,303 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Multi-threaded integration tests for S3 Encryption Client. + +These tests verify that the thread-local storage of encryption context +is properly isolated between threads when using a single S3EncryptionClient +instance across multiple threads. +""" + +import os +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime + +import boto3 + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.materials.kms_keyring import KmsKeyring + +bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") +region = os.environ.get("CI_AWS_REGION", "us-west-2") +kms_key_id = os.environ.get( + "CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" +) + + +def test_multithreaded_encryption_context_isolation(): + """Test that encryption context is properly isolated between threads. + + This test creates a single S3EncryptionClient instance and uses it + from multiple threads simultaneously, each with a different encryption + context. It verifies that: + 1. Each thread can encrypt with its own encryption context + 2. Each thread can decrypt only with the correct encryption context + 3. Thread-local storage doesn't leak between threads + """ + # Create a single S3EncryptionClient instance to be shared across threads + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + # Number of threads to test with + num_threads = 10 + results = {} + errors = [] + + def thread_worker(thread_id): + """Worker function for each thread.""" + try: + # Each thread has its own unique encryption context + encryption_context = { + "thread_id": str(thread_id), + "department": f"dept-{thread_id}", + "project": f"project-{thread_id}", + } + + # Unique key for this thread + key = f"multithread-test-{thread_id}-{datetime.now().strftime('%Y%m%d-%H%M%S-%f')}" + data = f"Thread {thread_id} test data with unique encryption context" + + # Encrypt with thread-specific encryption context + s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) + + # Decrypt with the SAME encryption context - should succeed + response = s3ec.get_object(Bucket=bucket, Key=key, EncryptionContext=encryption_context) + decrypted_data = response["Body"].read().decode("utf-8") + + if decrypted_data != data: + return { + "thread_id": thread_id, + "success": False, + "error": f"Data mismatch: expected '{data}', got '{decrypted_data}'", + } + + # Try to decrypt with a DIFFERENT encryption context - should fail + wrong_context = { + "thread_id": str(thread_id + 1000), + "department": "wrong-dept", + "project": "wrong-project", + } + + try: + s3ec.get_object(Bucket=bucket, Key=key, EncryptionContext=wrong_context) + return { + "thread_id": thread_id, + "success": False, + "error": "Decryption succeeded with wrong encryption context!", + } + except S3EncryptionClientError: + # Expected - decryption should fail with wrong context + pass + + # Try to decrypt with NO encryption context - should also fail + try: + s3ec.get_object(Bucket=bucket, Key=key) + return { + "thread_id": thread_id, + "success": False, + "error": "Decryption succeeded without encryption context!", + } + except S3EncryptionClientError: + # Expected - decryption should fail without context + pass + + return { + "thread_id": thread_id, + "success": True, + "key": key, + "encryption_context": encryption_context, + } + + except Exception as e: + return {"thread_id": thread_id, "success": False, "error": str(e)} + + # Execute threads concurrently + with ThreadPoolExecutor(max_workers=num_threads) as executor: + futures = [executor.submit(thread_worker, i) for i in range(num_threads)] + + for future in as_completed(futures): + result = future.result() + thread_id = result["thread_id"] + results[thread_id] = result + + if not result["success"]: + errors.append(f"Thread {thread_id}: {result['error']}") + + # Verify all threads succeeded + if errors: + print("Errors occurred during multi-threaded test:") + for error in errors: + print(f" - {error}") + raise RuntimeError(f"{len(errors)} thread(s) failed") + + print(f"Success! All {num_threads} threads completed successfully.") + print("Each thread:") + print(" - Encrypted with its own unique encryption context") + print(" - Decrypted successfully with the correct context") + print(" - Failed to decrypt with wrong context (as expected)") + print(" - Failed to decrypt without context (as expected)") + + +def test_multithreaded_rapid_context_switching(): + """Test rapid switching of encryption contexts in the same thread. + + This test verifies that encryption context is properly cleaned up + between operations within the same thread. + """ + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + num_iterations = 20 + errors = [] + + def rapid_context_worker(thread_id): + """Worker that rapidly switches between different encryption contexts.""" + try: + for i in range(num_iterations): + # Alternate between different encryption contexts + if i % 3 == 0: + encryption_context = {"iteration": str(i), "type": "typeA"} + elif i % 3 == 1: + encryption_context = {"iteration": str(i), "type": "typeB"} + else: + encryption_context = {"iteration": str(i), "type": "typeC"} + + key = ( + f"rapid-switch-t{thread_id}-i{i}-{datetime.now().strftime('%Y%m%d-%H%M%S-%f')}" + ) + data = f"Thread {thread_id} iteration {i}" + + # Encrypt + s3ec.put_object( + Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context + ) + + # Decrypt with correct context + response = s3ec.get_object( + Bucket=bucket, Key=key, EncryptionContext=encryption_context + ) + decrypted_data = response["Body"].read().decode("utf-8") + + if decrypted_data != data: + return { + "thread_id": thread_id, + "iteration": i, + "success": False, + "error": f"Data mismatch at iteration {i}", + } + + return {"thread_id": thread_id, "success": True} + + except Exception as e: + return {"thread_id": thread_id, "success": False, "error": str(e)} + + # Run multiple threads doing rapid context switching + num_threads = 5 + with ThreadPoolExecutor(max_workers=num_threads) as executor: + futures = [executor.submit(rapid_context_worker, i) for i in range(num_threads)] + + for future in as_completed(futures): + result = future.result() + if not result["success"]: + errors.append( + f"Thread {result['thread_id']}: {result.get('error', 'Unknown error')}" + ) + + if errors: + print("Errors occurred during rapid context switching test:") + for error in errors: + print(f" - {error}") + raise RuntimeError(f"{len(errors)} thread(s) failed") + + print(f"Success! {num_threads} threads completed {num_iterations} iterations each.") + print("Encryption context was properly isolated across rapid context switches.") + + +def test_multithreaded_mixed_with_and_without_context(): + """Test threads using encryption context mixed with threads not using it. + + This verifies that threads without encryption context don't interfere + with threads that use encryption context. + """ + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + errors = [] + + def worker_with_context(thread_id): + """Worker that uses encryption context.""" + try: + encryption_context = {"thread_id": str(thread_id), "has_context": "true"} + key = f"mixed-with-ctx-{thread_id}-{datetime.now().strftime('%Y%m%d-%H%M%S-%f')}" + data = f"Thread {thread_id} WITH context" + + s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) + + response = s3ec.get_object(Bucket=bucket, Key=key, EncryptionContext=encryption_context) + decrypted_data = response["Body"].read().decode("utf-8") + + if decrypted_data != data: + return {"thread_id": thread_id, "success": False, "error": "Data mismatch"} + + return {"thread_id": thread_id, "success": True, "type": "with_context"} + + except Exception as e: + return {"thread_id": thread_id, "success": False, "error": str(e)} + + def worker_without_context(thread_id): + """Worker that does NOT use encryption context.""" + try: + key = f"mixed-no-ctx-{thread_id}-{datetime.now().strftime('%Y%m%d-%H%M%S-%f')}" + data = f"Thread {thread_id} WITHOUT context" + + # No encryption context + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + + # No encryption context on decrypt either + response = s3ec.get_object(Bucket=bucket, Key=key) + decrypted_data = response["Body"].read().decode("utf-8") + + if decrypted_data != data: + return {"thread_id": thread_id, "success": False, "error": "Data mismatch"} + + return {"thread_id": thread_id, "success": True, "type": "without_context"} + + except Exception as e: + return {"thread_id": thread_id, "success": False, "error": str(e)} + + # Mix threads with and without encryption context + with ThreadPoolExecutor(max_workers=10) as executor: + futures = [] + + # Submit 5 threads with context + for i in range(5): + futures.append(executor.submit(worker_with_context, i)) + + # Submit 5 threads without context + for i in range(5, 10): + futures.append(executor.submit(worker_without_context, i)) + + for future in as_completed(futures): + result = future.result() + if not result["success"]: + errors.append( + f"Thread {result['thread_id']}: {result.get('error', 'Unknown error')}" + ) + + if errors: + print("Errors occurred during mixed context test:") + for error in errors: + print(f" - {error}") + raise RuntimeError(f"{len(errors)} thread(s) failed") + + print("Success! Mixed threads (with and without encryption context) completed successfully.") + print("Thread-local storage properly isolated context between threads.") diff --git a/test/test_decryption_materials_integration.py b/test/test_decryption_materials_integration.py index 1cfab083..35b7d9e8 100644 --- a/test/test_decryption_materials_integration.py +++ b/test/test_decryption_materials_integration.py @@ -1,7 +1,7 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock from src.s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager from src.s3_encryption.materials.encrypted_data_key import EncryptedDataKey @@ -30,17 +30,15 @@ def test_keyring_on_decrypt(self): encryption_context_from_request={"key2": "value2"}, ) - # Mock the validation method to return the materials - with patch.object(S3Keyring, "on_decrypt", return_value=materials) as mock_on_decrypt: - # Call on_decrypt - result = keyring.on_decrypt(materials, [edk]) + # Call on_decrypt + result = keyring.on_decrypt(materials, [edk]) - # Verify the result is a DecryptionMaterials instance - assert isinstance(result, DecryptionMaterials) - assert result.iv == b"initialization-vector" - assert result.encrypted_data_keys == [edk] - assert result.encryption_context_stored == {"key1": "value1"} - assert result.encryption_context_from_request == {"key2": "value2"} + # Verify the result is a DecryptionMaterials instance + assert isinstance(result, DecryptionMaterials) + assert result.iv == b"initialization-vector" + assert result.encrypted_data_keys == [edk] + assert result.encryption_context_stored == {"key1": "value1"} + assert result.encryption_context_from_request == {"key2": "value2"} def test_keyring_on_decrypt_default_enc_ctx(self): """Test that S3Keyring.on_decrypt properly handles DecryptionMaterials.""" @@ -63,16 +61,15 @@ def test_keyring_on_decrypt_default_enc_ctx(self): ) # Mock the validation method to return the materials - with patch.object(S3Keyring, "on_decrypt", return_value=materials) as mock_on_decrypt: - # Call on_decrypt - result = keyring.on_decrypt(materials, [edk]) - - # Verify the result is a DecryptionMaterials instance - assert isinstance(result, DecryptionMaterials) - assert result.iv == b"initialization-vector" - assert result.encrypted_data_keys == [edk] - assert result.encryption_context_stored == {} - assert result.encryption_context_from_request == {} + # Call on_decrypt + result = keyring.on_decrypt(materials, [edk]) + + # Verify the result is a DecryptionMaterials instance + assert isinstance(result, DecryptionMaterials) + assert result.iv == b"initialization-vector" + assert result.encrypted_data_keys == [edk] + assert result.encryption_context_stored == {} + assert result.encryption_context_from_request == {} def test_cmm_decrypt_materials_with_dict(self): """Test that DefaultCryptoMaterialsManager.decrypt_materials properly handles dictionary input.""" From b6647669ad6e2141f23c7e97c6d5a7952da9ae05 Mon Sep 17 00:00:00 2001 From: Tony Knapp <5892063+texastony@users.noreply.github.com> Date: Wed, 18 Feb 2026 11:35:47 -0600 Subject: [PATCH 56/81] ci(duvet): add spec and duvet for Python (#139) --- .duvet/.gitignore | 3 ++ .duvet/config.toml | 33 ++++++++++++++++++++ .github/workflows/all-ci.yml | 9 ++++++ .github/workflows/duvet-test-server.yml | 37 ++++++++++++++++------- .github/workflows/duvet.yml | 40 +++++++++++++++++++++++++ .github/workflows/test-server.yml | 2 +- .gitmodules | 4 +++ Makefile | 12 +++++++- specification | 1 + src/s3_encryption/pipelines.py | 7 +++++ 10 files changed, 136 insertions(+), 12 deletions(-) create mode 100644 .duvet/.gitignore create mode 100644 .duvet/config.toml create mode 100644 .github/workflows/duvet.yml create mode 160000 specification diff --git a/.duvet/.gitignore b/.duvet/.gitignore new file mode 100644 index 00000000..93956e36 --- /dev/null +++ b/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ \ No newline at end of file diff --git a/.duvet/config.toml b/.duvet/config.toml new file mode 100644 index 00000000..41d29170 --- /dev/null +++ b/.duvet/config.toml @@ -0,0 +1,33 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "src/**/*.py" +type = "implementation" +comment-style = { meta = "##=", content = "##%" } +[[source]] +pattern = "test/**/*.py" +type = "test" +comment-style = { meta = "##=", content = "##%" } + +# Include required specifications here +[[specification]] +source = "specification/s3-encryption/client.md" +[[specification]] +source = "specification/s3-encryption/decryption.md" +[[specification]] +source = "specification/s3-encryption/encryption.md" +[[specification]] +source = "specification/s3-encryption/key-commitment.md" +[[specification]] +source = "specification/s3-encryption/key-derivation.md" +[[specification]] +source = "specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "specification/s3-encryption/data-format/metadata-strategy.md" + +[report.html] +enabled = true + +# Enable snapshots to prevent requirement coverage regressions +[report.snapshot] +enabled = false diff --git a/.github/workflows/all-ci.yml b/.github/workflows/all-ci.yml index c65364b7..e35342fc 100644 --- a/.github/workflows/all-ci.yml +++ b/.github/workflows/all-ci.yml @@ -37,6 +37,15 @@ jobs: python-version: ${{ inputs.python-version || '3.11' }} secrets: inherit + run-duvet: + permissions: + id-token: write + contents: read + pages: write + name: Run Duvet + uses: ./.github/workflows/duvet.yml + secrets: inherit + run-duvet-test-server: permissions: id-token: write diff --git a/.github/workflows/duvet-test-server.yml b/.github/workflows/duvet-test-server.yml index f4bac5a8..f8f6e2ac 100644 --- a/.github/workflows/duvet-test-server.yml +++ b/.github/workflows/duvet-test-server.yml @@ -15,9 +15,31 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v5 - with: - submodules: true - token: ${{ secrets.PAT_FOR_SPEC }} + + # There are a lot of submodules here + # This initializes the checkouts in parallel (--jobs) + # rather than in series the way actions/checkout@v5 does it. + + - name: Get CPU count + id: cpu-count + run: echo "count=$(node -p 'require("os").cpus().length')" >> $GITHUB_OUTPUT + + - name: Setup git submodules with PAT + run: | + git config --global url."https://github.com/".insteadOf "git@github.com:" + git config --global credential.helper store + echo "https://x-token-auth:${{ secrets.PAT_FOR_SPEC }}@github.com" > ~/.git-credentials + + - name: Optimize git for performance + run: | + git config --global fetch.parallel ${{ steps.cpu-count.outputs.count }} + git config --global submodule.fetchJobs ${{ steps.cpu-count.outputs.count }} + git config --global remote.origin.tagOpt --no-tags + + - name: Checkout submodules with --jobs + run: | + git submodule update --init --depth 1 --single-branch --jobs ${{ steps.cpu-count.outputs.count }} test-server/ + - name: Checkout CPP code cpp-v3 uses: actions/checkout@v5 @@ -32,14 +54,9 @@ jobs: with: toolchain: stable - - name: Clone duvet repository - run: git clone https://github.com/awslabs/duvet.git /tmp/duvet - - name: Build and install duvet run: | - cd /tmp/duvet - cargo xtask build - cargo install --path ./duvet + cargo install duvet --locked - name: Run duvet if: always() @@ -49,7 +66,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: reports + name: test-server-reports include-hidden-files: true path: test-server/*-server/.duvet/reports/report.html diff --git a/.github/workflows/duvet.yml b/.github/workflows/duvet.yml new file mode 100644 index 00000000..eb7b49e2 --- /dev/null +++ b/.github/workflows/duvet.yml @@ -0,0 +1,40 @@ +name: duvet on the local S3EC-Python + +on: + workflow_call: + # Optional inputs that can be provided when calling this workflow + +jobs: + test: + runs-on: ubuntu-slim + permissions: + id-token: write + contents: read + pages: write + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Checkout specific specification + run: git submodule update --init --recursive specification + + - name: Setup Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: stable + + - name: Install duvet + run: | + cargo install duvet --locked + + - name: Run duvet + run: make duvet + + - name: Upload duvet reports + uses: actions/upload-artifact@v4 + with: + name: reports + include-hidden-files: true + path: .duvet/reports/report.html + diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml index 4fa10666..80991a99 100644 --- a/.github/workflows/test-server.yml +++ b/.github/workflows/test-server.yml @@ -46,7 +46,7 @@ jobs: - name: Checkout submodules with --jobs run: | - git submodule update --init --depth 1 --single-branch --jobs ${{ steps.cpu-count.outputs.count }} + git submodule update --init --depth 1 --single-branch --jobs ${{ steps.cpu-count.outputs.count }} test-server/ - name: Update cpp submodules recursively with --jobs run: | diff --git a/.gitmodules b/.gitmodules index 75e91f99..162cd457 100644 --- a/.gitmodules +++ b/.gitmodules @@ -46,3 +46,7 @@ path = test-server/cpp-v3-server/aws-sdk-cpp url = git@github.com:aws/aws-sdk-cpp.git branch = main +[submodule "specification"] + path = specification + url = https://github.com/awslabs/aws-encryption-sdk-specification.git + branch = tonyknap/s3ec-v3.0.1-candidate diff --git a/Makefile b/Makefile index 7db980c3..f295452b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: lint format test test-unit test-integration install # Default target -all: lint test +all: lint test duvet # Install dependencies install: @@ -37,3 +37,13 @@ clean: find . -type d -name .pytest_cache -exec rm -rf {} + find . -type d -name .coverage -exec rm -rf {} + find . -type f -name "*.pyc" -delete + rm -rf .duvet/reports/ .duvet/requirements/ + +duvet: | clean duvet-report + +duvet-report: + duvet report + +duvet-view-report-mac: + open .duvet/reports/report.html + diff --git a/specification b/specification new file mode 160000 index 00000000..7edabc2a --- /dev/null +++ b/specification @@ -0,0 +1 @@ +Subproject commit 7edabc2a69890e1119c49f948a086638182369a4 diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 6867ed7c..3a83a359 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -150,6 +150,13 @@ def decrypt(self, response, encryption_context=None): # Get decryption materials from the crypto materials manager dec_materials = self.cmm.decrypt_materials(dec_materials) + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=TODO + ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and + ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is NOT enabled, + ##% the S3EC MUST throw an error which details that client was + ##% not configured to decrypt objects with ALG_AES_256_CBC_IV16_NO_KDF. + aesgcm = AESGCM(dec_materials.plaintext_data_key) return aesgcm.decrypt(nonce=iv_bytes, data=encrypted_data, associated_data=None) From 4d8a1a64f5f470dd5025297537b13a12a2891b5a Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Tue, 3 Mar 2026 15:59:56 -0800 Subject: [PATCH 57/81] chore(duvet): annotate Keyring, S3Keyring, and KmsKeyring classes (#143) --- .duvet/config.toml | 10 + .../kms_keyring_exceptions.md | 214 ++++++++ pyproject.toml | 5 +- src/s3_encryption/materials/keyring.py | 61 ++- src/s3_encryption/materials/kms_keyring.py | 213 +++++--- test/test_decryption_materials_integration.py | 42 +- test/test_encryption_materials_integration.py | 21 +- test/test_kms_keyring.py | 469 ++++++++++++++++++ 8 files changed, 956 insertions(+), 79 deletions(-) create mode 100644 compliance_exceptions/kms_keyring_exceptions.md create mode 100644 test/test_kms_keyring.py diff --git a/.duvet/config.toml b/.duvet/config.toml index 41d29170..cb7abf7f 100644 --- a/.duvet/config.toml +++ b/.duvet/config.toml @@ -8,9 +8,19 @@ comment-style = { meta = "##=", content = "##%" } pattern = "test/**/*.py" type = "test" comment-style = { meta = "##=", content = "##%" } +[[source]] +pattern = "compliance_exceptions/**/*.md" +type = "exception" +comment-style = { meta = "##=", content = "##%" } # Include required specifications here [[specification]] +source = "specification/s3-encryption/materials/keyrings.md" +[[specification]] +source = "specification/s3-encryption/materials/s3-keyring.md" +[[specification]] +source = "specification/s3-encryption/materials/s3-kms-keyring.md" +[[specification]] source = "specification/s3-encryption/client.md" [[specification]] source = "specification/s3-encryption/decryption.md" diff --git a/compliance_exceptions/kms_keyring_exceptions.md b/compliance_exceptions/kms_keyring_exceptions.md new file mode 100644 index 00000000..0da55fb3 --- /dev/null +++ b/compliance_exceptions/kms_keyring_exceptions.md @@ -0,0 +1,214 @@ +# Compliance Exceptions for KMS Keyring Implementation + +## Summary + +The Python S3 Encryption Client implementation takes a pragmatic approach that: +1. Simplifies the keyring architecture by not implementing the full abstract method pattern (GenerateDataKey, EncryptDataKey, DecryptDataKey) +2. Defers validation to the AWS SDK where appropriate (key identifier validation) +3. Uses more efficient KMS API patterns (GenerateDataKey instead of separate Generate + Encrypt) +4. Omits optional features like custom User Agent strings (planned for future enhancement) + +## TODOs + +##= specification/s3-encryption/materials/s3-kms-keyring.md#kms-context +##= type=TODO +##% - A custom API Name or User Agent string SHOULD be provided in order to provide metrics on KMS calls associated with the S3 Encryption Client. + +##= specification/s3-encryption/materials/s3-kms-keyring.md#kmsv1 +##= type=TODO +##% - A custom API Name or User Agent string SHOULD be provided in order to provide metrics on KMS calls associated with the S3 Encryption Client. + +## Initialization Validation + +##= specification/s3-encryption/materials/s3-kms-keyring.md#initialization +##= type=exception +##% The KmsKeyring MAY validate that the AWS KMS key identifier is not null or empty. + +Justification: This validation is not implemented. The Python implementation relies on attrs field validation and KMS SDK to catch invalid key identifiers. + +--- + +##= specification/s3-encryption/materials/s3-kms-keyring.md#initialization +##= type=exception +##% If the KmsKeyring validates that the AWS KMS key identifier is not null or empty, then it MUST throw an exception. + +Justification: Not applicable since the MAY validation above is not implemented. If we don't validate, we don't need to throw an exception for validation failure. + +--- + +##= specification/s3-encryption/materials/s3-kms-keyring.md#initialization +##= type=exception +##% The KmsKeyring MAY validate that the AWS KMS key identifier is [a valid AWS KMS Key identifier](../../framework/aws-kms/aws-kms-key-arn.md#a-valid-aws-kms-identifier). + +Justification: This validation is not implemented. The Python implementation defers validation to the AWS KMS SDK, which will return an error if the key identifier is invalid. + +--- + +##= specification/s3-encryption/materials/s3-kms-keyring.md#initialization +##= type=exception +##% If the KmsKeyring validates that the AWS KMS key identifier is not a valid AWS KMS Key identifier, then it MUST throw an exception. + +Justification: Not applicable since the MAY validation above is not implemented. If we don't validate, we don't need to throw an exception for validation failure. + +--- + +## EncryptDataKey Method + +##= specification/s3-encryption/materials/s3-kms-keyring.md#encryptdatakey +##= type=exception +##% The KmsKeyring MUST implement the EncryptDataKey method. + +Justification: The Python implementation does not define a separate EncryptDataKey method. Instead, the encryption logic is directly implemented in the on_encrypt method using KMS GenerateDataKey API, which both generates and encrypts the data key in a single call. This is more efficient than the spec's pattern of separate Generate + Encrypt calls. + +--- + +##= specification/s3-encryption/materials/s3-kms-keyring.md#encryptdatakey +##= type=exception +##% The keyring MUST call [AWS KMS Encrypt](https://docs.aws.amazon.com/kms/latest/APIReference/API_Encrypt.html) using the configured AWS KMS client. + +Justification: The Python implementation uses KMS GenerateDataKey instead of KMS Encrypt. GenerateDataKey is more efficient as it generates and encrypts the data key in a single API call, rather than requiring separate generation and encryption operations. This reduces latency and API call count. + +--- + +##= specification/s3-encryption/materials/s3-kms-keyring.md#encryptdatakey +##= type=exception +##% - `KeyId` MUST be the configured AWS KMS key identifier. + +Justification: This requirement is for the KMS Encrypt API call. Since the Python implementation uses GenerateDataKey instead of Encrypt, this specific requirement doesn't apply. However, the KeyId parameter is correctly passed to GenerateDataKey. + +--- + +##= specification/s3-encryption/materials/s3-kms-keyring.md#encryptdatakey +##= type=exception +##% - `PlaintextDataKey` MUST be the plaintext data key in the [encryption materials](../structures.md#encryption-materials). + +Justification: The Python implementation uses KMS GenerateDataKey instead of Encrypt. GenerateDataKey generates the plaintext key itself, so there is no pre-existing plaintext data key to pass in. The Plaintext parameter doesn't exist in the GenerateDataKey API - instead, the API returns both the plaintext and encrypted versions of the newly generated key. + +--- + +##= specification/s3-encryption/materials/s3-kms-keyring.md#encryptdatakey +##= type=exception +##% - `EncryptionContext` MUST be the [encryption context](../structures.md#encryption-context) included in the input [encryption materials](../structures.md#encryption-materials). + +Justification: This requirement is for the KMS Encrypt API call. Since the Python implementation uses GenerateDataKey instead of Encrypt, this specific requirement doesn't apply. However, the EncryptionContext parameter is correctly passed to GenerateDataKey. + +--- + +##= specification/s3-encryption/materials/s3-kms-keyring.md#encryptdatakey +##= type=exception +##% - A custom API Name or User Agent string SHOULD be provided in order to provide metrics on KMS calls associated with the S3 Encryption Client. + +Justification: Custom User Agent strings are not currently implemented. This is a future enhancement for better observability and metrics tracking. The functionality works correctly without it, but metrics attribution to the S3 Encryption Client would be improved with this addition. + +--- + +##= specification/s3-encryption/materials/s3-kms-keyring.md#encryptdatakey +##= type=exception +##% If the call to [AWS KMS Encrypt](https://docs.aws.amazon.com/kms/latest/APIReference/API_Encrypt.html) does not succeed, OnEncrypt MUST fail. + +Justification: This requirement is for the KMS Encrypt API call. Since the Python implementation uses GenerateDataKey instead of Encrypt, this specific requirement doesn't apply. However, the implementation does correctly fail when GenerateDataKey fails. + +--- + +##= specification/s3-encryption/materials/s3-kms-keyring.md#encryptdatakey +##= type=exception +##% If the call to AWS KMS Encrypt is successful, OnEncrypt MUST return the `CiphertextBlob` as a collection of bytes. + +Justification: This requirement is for the KMS Encrypt API call. Since the Python implementation uses GenerateDataKey instead of Encrypt, this specific requirement doesn't apply. However, the implementation does correctly return the CiphertextBlob from GenerateDataKey's response. + +--- + +## DecryptDataKey Method Structure + +##= specification/s3-encryption/materials/s3-kms-keyring.md#kmsv1 +##= type=exception +##% - A custom API Name or User Agent string SHOULD be provided in order to provide metrics on KMS calls associated with the S3 Encryption Client. + +Justification: Custom User Agent strings are not currently implemented for KMS Decrypt calls. This is a future enhancement for better observability and metrics tracking. + +--- + +##= specification/s3-encryption/materials/s3-kms-keyring.md#kms-context +##= type=exception +##% - A custom API Name or User Agent string SHOULD be provided in order to provide metrics on KMS calls associated with the S3 Encryption Client. + +Justification: Custom User Agent strings are not currently implemented for KMS Decrypt calls in Kms+Context mode. This is a future enhancement for better observability and metrics tracking. + +--- + +## S3 Keyring Abstract Methods + +##= specification/s3-encryption/materials/s3-keyring.md#abstract-methods +##= type=exception +##% - The S3 Keyring MUST define an abstract method GenerateDataKey. + +Justification: The S3Keyring base class does not define abstract methods for GenerateDataKey, EncryptDataKey, or DecryptDataKey. The Python implementation uses a simpler design where concrete keyrings (like KmsKeyring) directly implement the on_encrypt and on_decrypt methods without the intermediate abstraction layer. This reduces complexity for the initial implementation. + +--- + +##= specification/s3-encryption/materials/s3-keyring.md#abstract-methods +##= type=exception +##% - The S3 Keyring MUST define an abstract method EncryptDataKey. + +Justification: The S3Keyring base class does not define abstract methods for GenerateDataKey, EncryptDataKey, or DecryptDataKey. +The Python implementation uses a simpler design where concrete keyrings (like KmsKeyring) directly implement the on_encrypt and on_decrypt methods without the intermediate abstraction layer. + +--- + +##= specification/s3-encryption/materials/s3-keyring.md#abstract-methods +##= type=exception +##% - The S3 Keyring MUST define an abstract method DecryptDataKey. + +Justification: The S3Keyring base class does not define abstract methods for GenerateDataKey, EncryptDataKey, or DecryptDataKey. +The Python implementation uses a simpler design where concrete keyrings (like KmsKeyring) directly implement the on_encrypt and on_decrypt methods without the intermediate abstraction layer. + +--- + +## S3 Keyring OnEncrypt Logic + +##= specification/s3-encryption/materials/s3-keyring.md#onencrypt +##= type=exception +##% If the Plaintext Data Key in the input EncryptionMaterials is null, the S3 Keyring MUST call the GenerateDataKey method using the materials. + +Justification: The S3Keyring base class does not implement this logic. The concrete KmsKeyring implementation directly calls KMS Encrypt in its on_encrypt method. +The specification's pattern of checking for null plaintext and conditionally calling GenerateDataKey is not followed; instead, the implementation always generates a new key. + +--- + +##= specification/s3-encryption/materials/s3-keyring.md#onencrypt +##= type=exception +##% If the materials returned from GenerateDataKey contain an EncryptedDataKey, the S3 Keyring MUST return the materials. + +Justification: Not applicable since the GenerateDataKey method pattern is not implemented. The KmsKeyring directly handles key generation and encryption in on_encrypt. + +--- + +##= specification/s3-encryption/materials/s3-keyring.md#onencrypt +##= type=exception +##% If the materials returned from GenerateDataKey do not contain an EncryptedDataKey, the S3 Keyring MUST call the EncryptDataKey method using the materials. + +Justification: Not applicable since the GenerateDataKey and EncryptDataKey method pattern is not implemented. +The KmsKeyring uses KMS GenerateDataKey which returns both plaintext and encrypted key in a single call. + +--- + +## S3 Keyring OnDecrypt Validations + +##= specification/s3-encryption/materials/s3-keyring.md#ondecrypt +##= type=exception +##% The S3 Keyring MAY validate that the Key Provider ID of the Encrypted Data Key matches the expected default Key Provider ID value. + +Justification: This optional validation is not implemented. +The Key Provider ID field is not used for anything in S3EC. + +--- + +##= specification/s3-encryption/materials/s3-keyring.md#ondecrypt +##= type=exception +##% The S3 Keyring MUST call the DecryptDataKey method using the materials and add the resulting plaintext data key to the materials. + +Justification: The S3Keyring base class does not implement this logic. +The concrete KmsKeyring implementation directly calls KMS Decrypt in its on_decrypt method rather than calling a separate DecryptDataKey method. +This is consistent with the simplified design that doesn't use the abstract method pattern. + +--- diff --git a/pyproject.toml b/pyproject.toml index b7489a03..5100857a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,10 @@ exclude = [".git", "__pycache__", "build", "dist"] [tool.ruff.lint] # Enable all rules by default, then configure specific rule settings below select = ["E", "F", "W", "I", "N", "D", "UP", "B", "A", "C4", "PT", "RET", "SIM", "ARG", "ERA"] -ignore = ["ARG002"] # Allow unused method arguments (e.g., **kwargs for API compatibility) +ignore = [ + "ARG002", # Allow unused method arguments (e.g., **kwargs for API compatibility) + "E501", # Line too long - Duvet annotations require long specification paths +] [tool.ruff.lint.pydocstyle] convention = "google" diff --git a/src/s3_encryption/materials/keyring.py b/src/s3_encryption/materials/keyring.py index 1e08cb18..d0ecd96b 100644 --- a/src/s3_encryption/materials/keyring.py +++ b/src/s3_encryption/materials/keyring.py @@ -14,6 +14,13 @@ from .materials import DecryptionMaterials, EncryptionMaterials +##= specification/s3-encryption/materials/keyrings.md#interface +##= type=implication +##% The Keyring interface and its operations SHOULD adhere to the naming conventions of the +##% implementation language. +##= specification/s3-encryption/materials/keyrings.md#supported-keyrings +##= type=implication +##% Note: A user MAY create their own custom keyring(s). @define class AbstractKeyring(abc.ABC): """Abstract base class for keyrings. @@ -22,8 +29,13 @@ class AbstractKeyring(abc.ABC): Concrete implementations handle specific key providers like KMS. """ + ##= specification/s3-encryption/materials/keyrings.md#interface + ##= type=implication + ##% The Keyring interface MUST include the OnEncrypt operation. + ##% The OnEncrypt operation MUST accept an instance of EncryptionMaterials as input. + ##% The OnEncrypt operation MUST return an instance of EncryptionMaterials as output. @abc.abstractmethod - def on_encrypt(self, enc_materials): + def on_encrypt(self, enc_materials) -> "EncryptionMaterials": """Process encryption materials. Args: @@ -34,8 +46,14 @@ def on_encrypt(self, enc_materials): """ pass + ##= specification/s3-encryption/materials/keyrings.md#interface + ##= type=implication + ##% The Keyring interface MUST include the OnDecrypt operation. + ##% The OnDecrypt operation MUST accept an instance of DecryptionMaterials and a collection + ##% of EncryptedDataKey instances as input. + ##% The OnDecrypt operation MUST return an instance of DecryptionMaterials as output. @abc.abstractmethod - def on_decrypt(self, dec_materials, encrypted_data_keys=None): + def on_decrypt(self, dec_materials, encrypted_data_keys=None) -> "DecryptionMaterials": """Decrypt one of the encrypted data keys and update dec_materials. Args: @@ -50,10 +68,19 @@ def on_decrypt(self, dec_materials, encrypted_data_keys=None): pass +##= specification/s3-encryption/materials/s3-keyring.md#overview +##= type=implication +##% The S3EC SHOULD implement an S3 Keyring to consolidate validation and other functionality +##% common to all S3 Keyrings. +##% If implemented, the S3 Keyring MUST implement the Keyring interface. @define class S3Keyring(AbstractKeyring): - """Base class for S3 encryption keyrings that provides common validation logic.""" + """Abstract class for S3EC keyrings that provides common validation logic.""" + ##= specification/s3-encryption/materials/s3-keyring.md#overview + ##= type=implication + ##% If implemented, the S3 Keyring MUST NOT be able to be instantiated as a Keyring instance. + @abc.abstractmethod def on_encrypt(self, enc_materials): """Validate encryption materials before encryption. @@ -79,6 +106,10 @@ def on_encrypt(self, enc_materials): return enc_materials + ##= specification/s3-encryption/materials/s3-keyring.md#overview + ##= type=implication + ##% If implemented, the S3 Keyring MUST NOT be able to be instantiated as a Keyring instance. + @abc.abstractmethod def on_decrypt(self, dec_materials, encrypted_data_keys=None): """Validate decryption materials before decryption. @@ -98,15 +129,33 @@ def on_decrypt(self, dec_materials, encrypted_data_keys=None): ) # Use encrypted_data_keys from parameters if provided, otherwise use from dec_materials + # TODO: This can probably be cleaned up, consult Java edks = ( encrypted_data_keys if encrypted_data_keys is not None else dec_materials.encrypted_data_keys ) - # Validate encrypted_data_keys - if edks is None or len(edks) == 0: - raise S3EncryptionClientError("No encrypted data keys provided") + if edks is None: + raise S3EncryptionClientError("No EncryptedDataKey provided on decrypt!") + + ##= specification/s3-encryption/materials/s3-keyring.md#ondecrypt + ##= type=implication + ##% If the input DecryptionMaterials contains a Plaintext Data Key, the S3 Keyring MUST + ##% throw an exception. + if dec_materials.plaintext_data_key is not None: + raise S3EncryptionClientError( + "Decryption materials already contains a plaintext data key." + ) + + ##= specification/s3-encryption/materials/s3-keyring.md#ondecrypt + ##= type=implication + ##% If the input collection of EncryptedDataKey instances contains any number of EDKs + ##% other than 1, the S3 Keyring MUST throw an exception. + if len(edks) != 1: + raise S3EncryptionClientError( + f"Only one encrypted data key is supported, found: {len(edks)}" + ) # Ensure encryption contexts are dictionaries if not isinstance(dec_materials.encryption_context_from_request, dict): diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index f8bc4997..a32493fc 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -17,6 +17,10 @@ KMS_V1_DEFAULT_KEY = "kms_cmk_id" +##= specification/s3-encryption/materials/s3-kms-keyring.md#interface +##= type=implication +##% The KmsKeyring MUST implement the [Keyring interface](keyrings.md#interface) and include +##% the behavior described in the [S3 Keyring](s3-keyring.md). @define class KmsKeyring(S3Keyring): """KMS implementation of the S3 keyring. @@ -29,8 +33,23 @@ class KmsKeyring(S3Keyring): enable_legacy_wrapping_algorithms (bool): Whether to enable legacy wrapping algorithms """ + ##= specification/s3-encryption/materials/s3-kms-keyring.md#initialization + ##= type=implementation + ##% On initialization, the caller MAY provide an AWS KMS SDK client instance. + ##= specification/s3-encryption/materials/s3-kms-keyring.md#initialization + ##= type=implication + ##% If the caller does not provide an AWS KMS SDK client instance or provides a null value, + ##% the KmsKeyring MUST create a default KMS client instance. kms_client: client.BaseClient = field() + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#initialization + ##= type=implementation + ##% On initialization, the caller MUST provide an AWS KMS key identifier. kms_key_id: str = field() + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#supported-wrapping-algorithm-modes + ##= type=implementation + ##% The KmsV1 mode MUST be only enabled when legacy wrapping algorithms are enabled. enable_legacy_wrapping_algorithms: bool = field(default=False) def on_encrypt(self, enc_materials): @@ -43,12 +62,25 @@ def on_encrypt(self, enc_materials): EncryptionMaterials: The processed encryption materials with KMS-generated keys """ try: - # Call parent class validation enc_materials = super().on_encrypt(enc_materials) encryption_context = enc_materials.encryption_context + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#supported-wrapping-algorithm-modes + ##= type=implementation + ##% The KmsKeyring MUST support encryption using Kms+Context mode. + ##= specification/s3-encryption/materials/s3-kms-keyring.md#supported-wrapping-algorithm-modes + ##= type=implementation + ##% The Kms+Context mode MUST be enabled as a fully-supported (non-legacy) wrapping + ##% algorithm. + ##= specification/s3-encryption/materials/s3-kms-keyring.md#supported-wrapping-algorithm-modes + ##= type=implication + ##% The KmsKeyring MUST NOT support encryption using KmsV1 mode. encryption_context["aws:x-amz-cek-alg"] = "AES/GCM/NoPadding" + # Python implementation uses KMS GenerateDataKey instead of the spec's + # EncryptDataKey pattern + # The spec is wrong and needs to be updated. response = self.kms_client.generate_data_key( KeyId=self.kms_key_id, KeySpec="AES_256", EncryptionContext=encryption_context ) @@ -62,6 +94,7 @@ def on_encrypt(self, enc_materials): enc_materials.plaintext_data_key = response["Plaintext"] return enc_materials except Exception: + # If KMS call fails, propagate the exception raise def on_decrypt(self, dec_materials, encrypted_data_keys=None): @@ -77,6 +110,10 @@ def on_decrypt(self, dec_materials, encrypted_data_keys=None): DecryptionMaterials: The updated dec_materials with the plaintext data key """ try: + ##= specification/s3-encryption/materials/s3-keyring.md#ondecrypt + ##= type=implication + ##% The OnDecrypt operation is responsible for ensuring that the DecryptionMaterials + ##% contain valid plaintext and encrypted data keys. # Call parent class validation dec_materials = super().on_decrypt(dec_materials, encrypted_data_keys) @@ -87,63 +124,123 @@ def on_decrypt(self, dec_materials, encrypted_data_keys=None): else dec_materials.encrypted_data_keys ) - # Try to decrypt each EDK until one succeeds - # TODO: probably just enforce |EDKs| == 1 and remove loop - last_exception = None - for edk in edks: - try: - edk_bytes = edk.encrypted_data_key - if edk.key_provider_info == "kms+context": - encryption_context_from_request = ( - dec_materials.encryption_context_from_request - ) - encryption_context_stored = dec_materials.encryption_context_stored - - # Default EC MUST NOT be passed in via request - if KMS_CONTEXT_DEFAULT_KEY in encryption_context_from_request: - raise S3EncryptionClientError( - f"{KMS_CONTEXT_DEFAULT_KEY} is a reserved key for the " - f"S3 encryption client" - ) - - # The stored EC, minus default key/values, MUST match provided EC - encryption_context_stored_copy = encryption_context_stored.copy() - encryption_context_stored_copy.pop(KMS_V1_DEFAULT_KEY, None) - encryption_context_stored_copy.pop(KMS_CONTEXT_DEFAULT_KEY, None) - - if encryption_context_stored_copy != encryption_context_from_request: - # TODO: modeled error - raise S3EncryptionClientError( - "Provided encryption context does not match information " - "retrieved from S3" - ) - - # Update decMaterials with the modified encryption context - elif edk.key_provider_info == "kms": - if not self.enable_legacy_wrapping_algorithms: - raise S3EncryptionClientError( - f"Enable legacy wrapping algorithms to use legacy key wrapping " - f"algorithm: {edk.key_provider_info}" - ) - else: - raise S3EncryptionClientError( - f"{edk.key_provider_info} is not a valid key wrapping algorithm!" - ) - - response = self.kms_client.decrypt( - KeyId=self.kms_key_id, - CiphertextBlob=edk_bytes, - EncryptionContext=dec_materials.encryption_context_stored, + # The parent class validation ensures there is exactly one EDK + edk = edks[0] + edk_bytes = edk.encrypted_data_key + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#supported-wrapping-algorithm-modes + ##= type=implementation + ##% The KmsKeyring MUST support decryption using Kms+Context mode. + ##= specification/s3-encryption/materials/s3-kms-keyring.md#decryptdatakey + ##= type=implementation + ##% The KmsKeyring MUST determine whether to decrypt using KmsV1 mode or + ##% Kms+Context mode. + ##= specification/s3-encryption/materials/s3-kms-keyring.md#decryptdatakey + ##= type=implementation + ##% If the Key Provider Info of the Encrypted Data Key is "kms+context", the + ##% KmsKeyring MUST attempt to decrypt using Kms+Context mode. + if edk.key_provider_info == "kms+context": + encryption_context_from_request = dec_materials.encryption_context_from_request + encryption_context_stored = dec_materials.encryption_context_stored + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kms-context + ##= type=implementation + ##% When decrypting using Kms+Context mode, the KmsKeyring MUST validate the + ##% provided (request) encryption context with the stored (materials) encryption + ##% context. + if KMS_CONTEXT_DEFAULT_KEY in encryption_context_from_request: + raise S3EncryptionClientError( + f"{KMS_CONTEXT_DEFAULT_KEY} is a reserved key for the " + f"S3 encryption client" ) - dec_materials.plaintext_data_key = response["Plaintext"] - return dec_materials - except Exception as e: - last_exception = e - continue - - # If we get here, none of the EDKs could be decrypted - if last_exception: - raise last_exception - raise S3EncryptionClientError("Failed to decrypt any of the encrypted data keys") + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kms-context + ##= type=implementation + ##% The stored encryption context with the two reserved keys removed MUST match + ##% the provided encryption context. + encryption_context_stored_copy = encryption_context_stored.copy() + encryption_context_stored_copy.pop(KMS_V1_DEFAULT_KEY, None) + encryption_context_stored_copy.pop(KMS_CONTEXT_DEFAULT_KEY, None) + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kms-context + ##= type=implementation + ##% If the stored encryption context with the two reserved keys removed does not + ##% match the provided encryption context, the KmsKeyring MUST throw an exception. + if encryption_context_stored_copy != encryption_context_from_request: + # TODO: modeled error + raise S3EncryptionClientError( + "Provided encryption context does not match information " + "retrieved from S3" + ) + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#decryptdatakey + ##= type=implication + ##% If the Key Provider Info of the Encrypted Data Key is "kms", the KmsKeyring + ##% MUST attempt to decrypt using KmsV1 mode. + elif edk.key_provider_info == "kms": + ##= specification/s3-encryption/materials/s3-kms-keyring.md#supported-wrapping-algorithm-modes + ##= type=implementation + ##% The KmsKeyring MUST support decryption using KmsV1 mode. + if not self.enable_legacy_wrapping_algorithms: + raise S3EncryptionClientError( + f"Enable legacy wrapping algorithms to use legacy key wrapping " + f"algorithm: {edk.key_provider_info}" + ) + else: + raise S3EncryptionClientError( + f"{edk.key_provider_info} is not a valid key wrapping algorithm!" + ) + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kmsv1 + ##= type=implementation + ##% To attempt to decrypt a particular [encrypted data key](../structures.md# + ##% encrypted-data-key), the KmsKeyring MUST call [AWS KMS Decrypt](https:// + ##% docs.aws.amazon.com/kms/latest/APIReference/API_Decrypt.html) with the + ##% configured AWS KMS client. + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kmsv1 + ##= type=implementation + ##% - `KeyId` MUST be the configured AWS KMS key identifier. + ##% - `CiphertextBlob` MUST be the [encrypted data key ciphertext]( + ##% ../structures.md#ciphertext). + ##% - `EncryptionContext` MUST be the [encryption context](../structures.md# + ##% encryption-context) included in the input [decryption materials]( + ##% ../structures.md#decryption-materials). + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kms-context + ##= type=implementation + ##% To attempt to decrypt a particular [encrypted data key](../structures.md# + ##% encrypted-data-key), the KmsKeyring MUST call [AWS KMS Decrypt](https:// + ##% docs.aws.amazon.com/kms/latest/APIReference/API_Decrypt.html) with the + ##% configured AWS KMS client. + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kms-context + ##= type=implication + ##% - `KeyId` MUST be the configured AWS KMS key identifier. + ##% - `CiphertextBlob` MUST be the [encrypted data key ciphertext]( + ##% ../structures.md#ciphertext). + ##% - `EncryptionContext` MUST be the [encryption context](../structures.md# + ##% encryption-context) included in the input [decryption materials]( + ##% ../structures.md#decryption-materials). + response = self.kms_client.decrypt( + KeyId=self.kms_key_id, + CiphertextBlob=edk_bytes, + EncryptionContext=dec_materials.encryption_context_stored, + ) + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kms-context + ##= type=implication + ##% The KmsKeyring MUST immediately return the plaintext as a collection of + ##% bytes. + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kmsv1 + ##= type=implication + ##% The KmsKeyring MUST immediately return the plaintext as a collection of + ##% bytes. + dec_materials.plaintext_data_key = response["Plaintext"] + return dec_materials except Exception: + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kmsv1 + ##= type=implementation + ##% If the KmsKeyring fails to successfully decrypt the [encrypted data key]( + ##% ../structures.md#encrypted-data-key), then it MUST throw an exception. + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kms-context + ##= type=implementation + ##% If the KmsKeyring fails to successfully decrypt the [encrypted data key]( + ##% ../structures.md#encrypted-data-key), then it MUST throw an exception. raise diff --git a/test/test_decryption_materials_integration.py b/test/test_decryption_materials_integration.py index 35b7d9e8..a4e45b4e 100644 --- a/test/test_decryption_materials_integration.py +++ b/test/test_decryption_materials_integration.py @@ -5,15 +5,24 @@ from src.s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager from src.s3_encryption.materials.encrypted_data_key import EncryptedDataKey -from src.s3_encryption.materials.keyring import S3Keyring +from src.s3_encryption.materials.kms_keyring import KmsKeyring from src.s3_encryption.materials.materials import DecryptionMaterials class TestDecryptionMaterialsIntegration: def test_keyring_on_decrypt(self): - """Test that S3Keyring.on_decrypt properly handles DecryptionMaterials.""" + """Test that KmsKeyring.on_decrypt properly handles DecryptionMaterials.""" + # Create a mock KMS client + mock_kms_client = MagicMock() + mock_kms_client.decrypt.return_value = { + "Plaintext": b"plaintext-data-key", + } + # Create a keyring - keyring = S3Keyring() + keyring = KmsKeyring( + kms_client=mock_kms_client, + kms_key_id="arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012", + ) # Create an encrypted data key edk = EncryptedDataKey( @@ -22,12 +31,13 @@ def test_keyring_on_decrypt(self): encrypted_data_key=b"encrypted-data-key", ) - # Create decryption materials + # Create decryption materials with matching encryption contexts + # The stored context includes the reserved key, the request context should match (minus reserved keys) materials = DecryptionMaterials( iv=b"initialization-vector", encrypted_data_keys=[edk], - encryption_context_stored={"key1": "value1"}, - encryption_context_from_request={"key2": "value2"}, + encryption_context_stored={"key1": "value1", "aws:x-amz-cek-alg": "AES/GCM/NoPadding"}, + encryption_context_from_request={"key1": "value1"}, ) # Call on_decrypt @@ -37,13 +47,25 @@ def test_keyring_on_decrypt(self): assert isinstance(result, DecryptionMaterials) assert result.iv == b"initialization-vector" assert result.encrypted_data_keys == [edk] - assert result.encryption_context_stored == {"key1": "value1"} - assert result.encryption_context_from_request == {"key2": "value2"} + assert result.encryption_context_stored == { + "key1": "value1", + "aws:x-amz-cek-alg": "AES/GCM/NoPadding", + } + assert result.encryption_context_from_request == {"key1": "value1"} def test_keyring_on_decrypt_default_enc_ctx(self): - """Test that S3Keyring.on_decrypt properly handles DecryptionMaterials.""" + """Test that KmsKeyring.on_decrypt properly handles DecryptionMaterials.""" + # Create a mock KMS client + mock_kms_client = MagicMock() + mock_kms_client.decrypt.return_value = { + "Plaintext": b"plaintext-data-key", + } + # Create a keyring - keyring = S3Keyring() + keyring = KmsKeyring( + kms_client=mock_kms_client, + kms_key_id="arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012", + ) # Create an encrypted data key edk = EncryptedDataKey( diff --git a/test/test_encryption_materials_integration.py b/test/test_encryption_materials_integration.py index 989d17d8..4b6bfefd 100644 --- a/test/test_encryption_materials_integration.py +++ b/test/test_encryption_materials_integration.py @@ -5,15 +5,25 @@ from src.s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager from src.s3_encryption.materials.encrypted_data_key import EncryptedDataKey -from src.s3_encryption.materials.keyring import S3Keyring +from src.s3_encryption.materials.kms_keyring import KmsKeyring from src.s3_encryption.materials.materials import EncryptionMaterials class TestEncryptionMaterialsIntegration: def test_keyring_on_encrypt(self): - """Test that S3Keyring.on_encrypt properly handles EncryptionMaterials.""" + """Test that KmsKeyring.on_encrypt properly handles EncryptionMaterials.""" + # Create a mock KMS client + mock_kms_client = MagicMock() + mock_kms_client.generate_data_key.return_value = { + "CiphertextBlob": b"encrypted-data-key", + "Plaintext": b"plaintext-data-key", + } + # Create a keyring - keyring = S3Keyring() + keyring = KmsKeyring( + kms_client=mock_kms_client, + kms_key_id="arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012", + ) # Create encryption materials materials = EncryptionMaterials(encryption_context={"key1": "value1"}) @@ -23,7 +33,10 @@ def test_keyring_on_encrypt(self): # Verify the result is an EncryptionMaterials instance assert isinstance(result, EncryptionMaterials) - assert result.encryption_context == {"key1": "value1"} + assert result.encryption_context == { + "key1": "value1", + "aws:x-amz-cek-alg": "AES/GCM/NoPadding", + } def test_cmm_get_encryption_materials_with_dict(self): """Test that DefaultCryptoMaterialsManager.get_encryption_materials properly handles dictionary input.""" diff --git a/test/test_kms_keyring.py b/test/test_kms_keyring.py new file mode 100644 index 00000000..d613cbf9 --- /dev/null +++ b/test/test_kms_keyring.py @@ -0,0 +1,469 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for KMS keyring implementation.""" + +from unittest.mock import MagicMock + +import pytest + +from src.s3_encryption.exceptions import S3EncryptionClientError +from src.s3_encryption.materials.encrypted_data_key import EncryptedDataKey +from src.s3_encryption.materials.kms_keyring import KmsKeyring +from src.s3_encryption.materials.materials import DecryptionMaterials, EncryptionMaterials + + +class TestKmsKeyringInitialization: + """Tests for KMS keyring initialization.""" + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#initialization + ##= type=test + ##% On initialization, the caller MUST provide an AWS KMS key identifier. + def test_initialization_with_required_parameters(self): + """Test that KMS keyring can be initialized with required parameters.""" + mock_kms_client = MagicMock() + kms_key_id = "arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012" + + keyring = KmsKeyring(kms_client=mock_kms_client, kms_key_id=kms_key_id) + + assert keyring.kms_client == mock_kms_client + assert keyring.kms_key_id == kms_key_id + assert keyring.enable_legacy_wrapping_algorithms is False + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#initialization + ##= type=test + ##% On initialization, the caller MAY provide an AWS KMS SDK client instance. + def test_initialization_with_kms_client(self): + """Test that KMS keyring accepts a KMS client instance.""" + mock_kms_client = MagicMock() + kms_key_id = "test-key-id" + + keyring = KmsKeyring(kms_client=mock_kms_client, kms_key_id=kms_key_id) + + assert keyring.kms_client == mock_kms_client + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#supported-wrapping-algorithm-modes + ##= type=test + ##% The KmsV1 mode MUST be only enabled when legacy wrapping algorithms are enabled. + def test_initialization_with_legacy_wrapping_algorithms(self): + """Test that legacy wrapping algorithms can be enabled.""" + mock_kms_client = MagicMock() + kms_key_id = "test-key-id" + + keyring = KmsKeyring( + kms_client=mock_kms_client, + kms_key_id=kms_key_id, + enable_legacy_wrapping_algorithms=True, + ) + + assert keyring.enable_legacy_wrapping_algorithms is True + + +class TestKmsKeyringOnEncrypt: + """Tests for KMS keyring encryption operations.""" + + def test_on_encrypt_returns_encryption_materials(self): + """Test that on_encrypt returns EncryptionMaterials.""" + mock_kms_client = MagicMock() + mock_kms_client.generate_data_key.return_value = { + "CiphertextBlob": b"encrypted-key", + "Plaintext": b"plaintext-key", + } + + keyring = KmsKeyring(kms_client=mock_kms_client, kms_key_id="test-key-id") + enc_materials = EncryptionMaterials(encryption_context={"key": "value"}) + + result = keyring.on_encrypt(enc_materials) + + assert isinstance(result, EncryptionMaterials) + + def test_on_encrypt_calls_kms_generate_data_key(self): + """Test that on_encrypt calls KMS generate_data_key.""" + mock_kms_client = MagicMock() + mock_kms_client.generate_data_key.return_value = { + "CiphertextBlob": b"encrypted-key", + "Plaintext": b"plaintext-key", + } + + keyring = KmsKeyring(kms_client=mock_kms_client, kms_key_id="test-key-id") + enc_materials = EncryptionMaterials(encryption_context={"key": "value"}) + + keyring.on_encrypt(enc_materials) + + mock_kms_client.generate_data_key.assert_called_once() + + def test_on_encrypt_uses_correct_kms_parameters(self): + """Test that on_encrypt uses correct KMS parameters.""" + mock_kms_client = MagicMock() + mock_kms_client.generate_data_key.return_value = { + "CiphertextBlob": b"encrypted-key", + "Plaintext": b"plaintext-key", + } + + kms_key_id = "test-key-id" + keyring = KmsKeyring(kms_client=mock_kms_client, kms_key_id=kms_key_id) + encryption_context = {"key": "value"} + enc_materials = EncryptionMaterials(encryption_context=encryption_context) + + keyring.on_encrypt(enc_materials) + + call_args = mock_kms_client.generate_data_key.call_args + assert call_args.kwargs["KeyId"] == kms_key_id + assert "aws:x-amz-cek-alg" in call_args.kwargs["EncryptionContext"] + assert call_args.kwargs["EncryptionContext"]["key"] == "value" + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#supported-wrapping-algorithm-modes + ##= type=test + ##% The KmsKeyring MUST support encryption using Kms+Context mode. + def test_on_encrypt_adds_kms_context_algorithm(self): + """Test that on_encrypt adds the Kms+Context algorithm to encryption context.""" + mock_kms_client = MagicMock() + mock_kms_client.generate_data_key.return_value = { + "CiphertextBlob": b"encrypted-key", + "Plaintext": b"plaintext-key", + } + + keyring = KmsKeyring(kms_client=mock_kms_client, kms_key_id="test-key-id") + enc_materials = EncryptionMaterials(encryption_context={}) + + result = keyring.on_encrypt(enc_materials) + + call_args = mock_kms_client.generate_data_key.call_args + assert call_args.kwargs["EncryptionContext"]["aws:x-amz-cek-alg"] == "AES/GCM/NoPadding" + + def test_on_encrypt_sets_encrypted_data_key(self): + """Test that on_encrypt sets the encrypted data key from KMS response.""" + mock_kms_client = MagicMock() + ciphertext_blob = b"encrypted-key-from-kms" + plaintext = b"plaintext-key-from-kms" + mock_kms_client.generate_data_key.return_value = { + "CiphertextBlob": ciphertext_blob, + "Plaintext": plaintext, + } + + keyring = KmsKeyring(kms_client=mock_kms_client, kms_key_id="test-key-id") + enc_materials = EncryptionMaterials(encryption_context={}) + + result = keyring.on_encrypt(enc_materials) + + assert result.encrypted_data_key is not None + assert result.encrypted_data_key.encrypted_data_key == ciphertext_blob + assert result.encrypted_data_key.key_provider_info == "kms+context" + assert result.plaintext_data_key == plaintext + + def test_on_encrypt_fails_when_kms_fails(self): + """Test that on_encrypt fails when KMS call fails.""" + mock_kms_client = MagicMock() + mock_kms_client.generate_data_key.side_effect = Exception("KMS error") + + keyring = KmsKeyring(kms_client=mock_kms_client, kms_key_id="test-key-id") + enc_materials = EncryptionMaterials(encryption_context={}) + + with pytest.raises(Exception): + keyring.on_encrypt(enc_materials) + + +class TestKmsKeyringOnDecrypt: + """Tests for KMS keyring decryption operations.""" + + def test_on_decrypt_returns_decryption_materials(self): + """Test that on_decrypt returns DecryptionMaterials.""" + mock_kms_client = MagicMock() + mock_kms_client.decrypt.return_value = {"Plaintext": b"plaintext-key"} + + keyring = KmsKeyring(kms_client=mock_kms_client, kms_key_id="test-key-id") + edk = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms+context", + encrypted_data_key=b"encrypted-key", + ) + dec_materials = DecryptionMaterials( + iv=b"initialization-vector", + encrypted_data_keys=[edk], + encryption_context_stored={"aws:x-amz-cek-alg": "AES/GCM/NoPadding"}, + encryption_context_from_request={}, + ) + + result = keyring.on_decrypt(dec_materials) + + assert isinstance(result, DecryptionMaterials) + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#decryptdatakey + ##= type=test + ##% The KmsKeyring MUST determine whether to decrypt using KmsV1 mode or Kms+Context mode. + ##= specification/s3-encryption/materials/s3-kms-keyring.md#supported-wrapping-algorithm-modes + ##= type=test + ##% The KmsKeyring MUST support decryption using Kms+Context mode. + ##% The Kms+Context mode MUST be enabled as a fully-supported (non-legacy) wrapping algorithm. + def test_on_decrypt_with_kms_context_mode(self): + """Test that on_decrypt handles kms+context mode.""" + mock_kms_client = MagicMock() + mock_kms_client.decrypt.return_value = {"Plaintext": b"plaintext-key"} + + keyring = KmsKeyring(kms_client=mock_kms_client, kms_key_id="test-key-id") + edk = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms+context", + encrypted_data_key=b"encrypted-key", + ) + dec_materials = DecryptionMaterials( + iv=b"initialization-vector", + encrypted_data_keys=[edk], + encryption_context_stored={"aws:x-amz-cek-alg": "AES/GCM/NoPadding"}, + encryption_context_from_request={}, + ) + + result = keyring.on_decrypt(dec_materials) + + assert result.plaintext_data_key == b"plaintext-key" + mock_kms_client.decrypt.assert_called_once() + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#decryptdatakey + ##= type=test + ##% If the Key Provider Info of the Encrypted Data Key is "kms+context", the KmsKeyring MUST attempt to decrypt using Kms+Context mode. + def test_on_decrypt_validates_encryption_context(self): + """Test that on_decrypt validates encryption context.""" + mock_kms_client = MagicMock() + mock_kms_client.decrypt.return_value = {"Plaintext": b"plaintext-key"} + + keyring = KmsKeyring(kms_client=mock_kms_client, kms_key_id="test-key-id") + edk = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms+context", + encrypted_data_key=b"encrypted-key", + ) + dec_materials = DecryptionMaterials( + iv=b"initialization-vector", + encrypted_data_keys=[edk], + encryption_context_stored={ + "aws:x-amz-cek-alg": "AES/GCM/NoPadding", + "custom-key": "custom-value", + }, + encryption_context_from_request={"custom-key": "custom-value"}, + ) + + result = keyring.on_decrypt(dec_materials) + + assert result.plaintext_data_key == b"plaintext-key" + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kms-context + ##= type=test + ##% When decrypting using Kms+Context mode, the KmsKeyring MUST validate the provided (request) encryption context with the stored (materials) encryption context. + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kms-context + ##= type=test + ##% If the stored encryption context with the two reserved keys removed does not match the provided encryption context, the KmsKeyring MUST throw an exception. + def test_on_decrypt_fails_with_mismatched_encryption_context(self): + """Test that on_decrypt fails when encryption contexts don't match.""" + mock_kms_client = MagicMock() + + keyring = KmsKeyring(kms_client=mock_kms_client, kms_key_id="test-key-id") + edk = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms+context", + encrypted_data_key=b"encrypted-key", + ) + dec_materials = DecryptionMaterials( + iv=b"initialization-vector", + encrypted_data_keys=[edk], + encryption_context_stored={ + "aws:x-amz-cek-alg": "AES/GCM/NoPadding", + "custom-key": "stored-value", + }, + encryption_context_from_request={"custom-key": "different-value"}, + ) + + with pytest.raises(S3EncryptionClientError) as exc_info: + keyring.on_decrypt(dec_materials) + + assert "does not match" in str(exc_info.value) + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kms-context + ##= type=test + ##% The stored encryption context with the two reserved keys removed MUST match the provided encryption context. + def test_on_decrypt_rejects_reserved_key_in_request_context(self): + """Test that on_decrypt rejects reserved keys in request encryption context.""" + mock_kms_client = MagicMock() + + keyring = KmsKeyring(kms_client=mock_kms_client, kms_key_id="test-key-id") + edk = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms+context", + encrypted_data_key=b"encrypted-key", + ) + dec_materials = DecryptionMaterials( + iv=b"initialization-vector", + encrypted_data_keys=[edk], + encryption_context_stored={"aws:x-amz-cek-alg": "AES/GCM/NoPadding"}, + encryption_context_from_request={"aws:x-amz-cek-alg": "AES/GCM/NoPadding"}, + ) + + with pytest.raises(S3EncryptionClientError) as exc_info: + keyring.on_decrypt(dec_materials) + + assert "reserved key" in str(exc_info.value) + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#decryptdatakey + ##= type=test + ##% If the Key Provider Info of the Encrypted Data Key is "kms", the KmsKeyring MUST attempt to decrypt using KmsV1 mode. + def test_on_decrypt_with_kms_v1_mode(self): + """Test that on_decrypt handles KmsV1 mode when legacy algorithms are enabled.""" + mock_kms_client = MagicMock() + mock_kms_client.decrypt.return_value = {"Plaintext": b"plaintext-key"} + + kms_key_id = "test-key-id" + encrypted_key = b"encrypted-key" + encryption_context_stored = {"foo": "bar"} + + keyring = KmsKeyring( + kms_client=mock_kms_client, + kms_key_id=kms_key_id, + enable_legacy_wrapping_algorithms=True, + ) + edk = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms", + encrypted_data_key=encrypted_key, + ) + dec_materials = DecryptionMaterials( + iv=b"initialization-vector", + encrypted_data_keys=[edk], + encryption_context_stored=encryption_context_stored, + encryption_context_from_request={}, + ) + + result = keyring.on_decrypt(dec_materials) + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kmsv1 + ##= type=test + ##% To attempt to decrypt a particular [encrypted data key](../structures.md#encrypted-data-key), the KmsKeyring MUST call [AWS KMS Decrypt](https://docs.aws.amazon.com/kms/latest/APIReference/API_Decrypt.html) with the configured AWS KMS client. + call_args = mock_kms_client.decrypt.call_args + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kmsv1 + ##= type=test + ##% - `KeyId` MUST be the configured AWS KMS key identifier. + assert call_args.kwargs["KeyId"] == kms_key_id + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kmsv1 + ##= type=test + ##% - `CiphertextBlob` MUST be the [encrypted data key ciphertext](../structures.md#ciphertext). + assert call_args.kwargs["CiphertextBlob"] == encrypted_key + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kmsv1 + ##= type=test + ##% - `EncryptionContext` MUST be the [encryption context](../structures.md#encryption-context) included in the input [decryption materials](../structures.md#decryption-materials). + assert call_args.kwargs["EncryptionContext"] == encryption_context_stored + assert result.plaintext_data_key == b"plaintext-key" + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#supported-wrapping-algorithm-modes + ##= type=test + ##% The KmsKeyring MUST support decryption using KmsV1 mode. + def test_on_decrypt_rejects_kms_v1_when_legacy_disabled(self): + """Test that on_decrypt rejects KmsV1 mode when legacy algorithms are disabled.""" + mock_kms_client = MagicMock() + + keyring = KmsKeyring( + kms_client=mock_kms_client, + kms_key_id="test-key-id", + enable_legacy_wrapping_algorithms=False, + ) + edk = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms", + encrypted_data_key=b"encrypted-key", + ) + dec_materials = DecryptionMaterials( + iv=b"initialization-vector", + encrypted_data_keys=[edk], + encryption_context_stored={}, + encryption_context_from_request={}, + ) + + with pytest.raises(S3EncryptionClientError) as exc_info: + keyring.on_decrypt(dec_materials) + + assert "legacy wrapping algorithms" in str(exc_info.value) + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kms-context + ##= type=test + ##% To attempt to decrypt a particular [encrypted data key](../structures.md#encrypted-data-key), the KmsKeyring MUST call [AWS KMS Decrypt](https://docs.aws.amazon.com/kms/latest/APIReference/API_Decrypt.html) with the configured AWS KMS client. + def test_on_decrypt_uses_correct_kms_parameters(self): + """Test that on_decrypt uses correct KMS parameters.""" + mock_kms_client = MagicMock() + mock_kms_client.decrypt.return_value = {"Plaintext": b"plaintext-key"} + + kms_key_id = "test-key-id" + keyring = KmsKeyring(kms_client=mock_kms_client, kms_key_id=kms_key_id) + encrypted_key = b"encrypted-key-bytes" + edk = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms+context", + encrypted_data_key=encrypted_key, + ) + encryption_context_stored = {"aws:x-amz-cek-alg": "AES/GCM/NoPadding"} + dec_materials = DecryptionMaterials( + iv=b"initialization-vector", + encrypted_data_keys=[edk], + encryption_context_stored=encryption_context_stored, + encryption_context_from_request={}, + ) + + keyring.on_decrypt(dec_materials) + + call_args = mock_kms_client.decrypt.call_args + assert call_args.kwargs["KeyId"] == kms_key_id + assert call_args.kwargs["CiphertextBlob"] == encrypted_key + assert call_args.kwargs["EncryptionContext"] == encryption_context_stored + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kmsv1 + ##= type=test + ##% If the KmsKeyring fails to successfully decrypt the [encrypted data key](../structures.md#encrypted-data-key), then it MUST throw an exception. + def test_on_decrypt_fails_when_kms_v1_fails(self): + """Test that on_decrypt fails when KMS call fails.""" + mock_kms_client = MagicMock() + kms_exception = Exception("KMS decrypt error") + mock_kms_client.decrypt.side_effect = kms_exception + + keyring = KmsKeyring( + kms_client=mock_kms_client, + kms_key_id="test-key-id", + enable_legacy_wrapping_algorithms=True, + ) + edk = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms", + encrypted_data_key=b"encrypted-key", + ) + dec_materials = DecryptionMaterials( + iv=b"initialization-vector", + encrypted_data_keys=[edk], + encryption_context_stored={}, + encryption_context_from_request={}, + ) + + with pytest.raises(Exception, match="KMS decrypt error") as exc_info: + keyring.on_decrypt(dec_materials) + + assert exc_info.value is kms_exception + + ##= specification/s3-encryption/materials/s3-kms-keyring.md#kms-context + ##= type=test + ##% If the KmsKeyring fails to successfully decrypt the [encrypted data key](../structures.md#encrypted-data-key), then it MUST throw an exception. + def test_on_decrypt_fails_when_kms_fails(self): + """Test that on_decrypt fails when KMS call fails.""" + mock_kms_client = MagicMock() + kms_exception = Exception("KMS decrypt error") + mock_kms_client.decrypt.side_effect = kms_exception + + keyring = KmsKeyring(kms_client=mock_kms_client, kms_key_id="test-key-id") + edk = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms+context", + encrypted_data_key=b"encrypted-key", + ) + dec_materials = DecryptionMaterials( + iv=b"initialization-vector", + encrypted_data_keys=[edk], + encryption_context_stored={"aws:x-amz-cek-alg": "AES/GCM/NoPadding"}, + encryption_context_from_request={}, + ) + + with pytest.raises(Exception, match="KMS decrypt error") as exc_info: + keyring.on_decrypt(dec_materials) + + assert exc_info.value is kms_exception From f8a6c4c64efb536eac42a9277ebe55e7c92b0ceb Mon Sep 17 00:00:00 2001 From: Tony Knapp <5892063+texastony@users.noreply.github.com> Date: Wed, 4 Mar 2026 15:05:06 -0600 Subject: [PATCH 58/81] feat(metadata): Add Instruction File Support (#140) feat(instruction-file): add instruction file support for S3 Encryption Client Add support for reading encrypted objects that store metadata in instruction files, enabling cross-language compatibility with Java, Ruby, PHP, Go, and .NET S3 Encryption Client implementations. Key changes: - Add ObjectMetadata V3 format support with compressed metadata keys - Add version detection methods (is_v1_format, is_v2_format, is_v3_format) - Refactor decrypt pipeline to use version detection and extract per-version logic - Implement instruction file fetching with thread-local mode flag - Parse and validate instruction file JSON with S3EC metadata key validation - Add configurable instruction_file_suffix (default: ".instruction") - Add 7 duvet specification citations with implementation and test annotations - Add CDK infrastructure for static cross-language test objects bucket Testing: - Unit tests for metadata parsing, pipeline decryption, and event handler validation - Integration tests using static test objects created by Java S3EC - V2 instruction file and custom suffix tests active and passing - V1 CBC and V3 tests skipped pending decryption implementation - Negative test for invalid instruction files --- cdk/lib/cdk-stack.ts | 12 + cdk/package-lock.json | 262 ++++++++++++------ cdk/package.json | 4 +- pyproject.toml | 2 + src/s3_encryption/__init__.py | 80 +++++- src/s3_encryption/instruction_file.py | 117 ++++++++ src/s3_encryption/metadata.py | 166 +++++++++++ src/s3_encryption/pipelines.py | 139 +++++++--- .../test_i_s3_encryption_instruction_file.py | 151 ++++++++++ test/test_metadata.py | 157 +++++++++++ test/test_pipelines.py | 241 ++++++++++++++++ test/test_s3_encryption_client_plugin.py | 141 ++++++++++ 12 files changed, 1333 insertions(+), 139 deletions(-) create mode 100644 src/s3_encryption/instruction_file.py create mode 100644 test/integration/test_i_s3_encryption_instruction_file.py create mode 100644 test/test_pipelines.py create mode 100644 test/test_s3_encryption_client_plugin.py diff --git a/cdk/lib/cdk-stack.ts b/cdk/lib/cdk-stack.ts index 1fad4b74..b5a28084 100644 --- a/cdk/lib/cdk-stack.ts +++ b/cdk/lib/cdk-stack.ts @@ -91,6 +91,16 @@ export class S3ECPythonGithub extends cdk.Stack { } ) + // New bucket for static test objects + const S3ECStaticTestObjectsBucket = new Bucket( + this, + "S3ECStaticTestObjectsBucket", + { + bucketName: "s3ec-static-test-objects", + blockPublicAccess: new BlockPublicAccess(AccessConfiguration) + } + ) + // S3 bucket policy const S3ECGithubS3BucketPolicy = new ManagedPolicy( this, @@ -110,6 +120,7 @@ export class S3ECPythonGithub extends cdk.Stack { resources: [ S3ECGithubTestS3Bucket.bucketArn + "/*", // object-level permissions need this extra path S3ECTestServerGithubBucket.bucketArn + "/*", // Add permissions for the new test-server bucket + S3ECStaticTestObjectsBucket.bucketArn + "/*", // Add permissions for static test objects bucket "arn:aws:s3:::aws-net-sdk-*/*" // permission for object inside S3EC .net bucket. For S3EC-NET repo ], }), @@ -125,6 +136,7 @@ export class S3ECPythonGithub extends cdk.Stack { resources: [ S3ECGithubTestS3Bucket.bucketArn, S3ECTestServerGithubBucket.bucketArn, // Add permissions for the new test-server bucket + S3ECStaticTestObjectsBucket.bucketArn, // Add permissions for static test objects bucket "arn:aws:s3:::aws-net-sdk-*", // permission for S3EC .net bucket. For S3EC-NET repo ], }), diff --git a/cdk/package-lock.json b/cdk/package-lock.json index 4f44562c..fa174491 100644 --- a/cdk/package-lock.json +++ b/cdk/package-lock.json @@ -8,7 +8,7 @@ "name": "cdk", "version": "0.1.0", "dependencies": { - "aws-cdk-lib": "2.92.0", + "aws-cdk-lib": "^2.240.0", "constructs": "^10.0.0", "source-map-support": "^0.5.21" }, @@ -40,16 +40,9 @@ } }, "node_modules/@aws-cdk/asset-awscli-v1": { - "version": "2.2.247", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.247.tgz", - "integrity": "sha512-PGFzztdu5YozUgoUd8gq5qi1FR3EYMjNrl5JFrAlYh2w1PcTfExEwqDzZy9z6uzogEJKwQJDgyhWe+OcZzQqFg==", - "license": "Apache-2.0" - }, - "node_modules/@aws-cdk/asset-kubectl-v20": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.4.tgz", - "integrity": "sha512-Ps2MkmjYgMyflagqQ4dgTElc7Vwpqj8spw8dQVFiSeaaMPsuDSNsPax3/HjuDuwqsmLdaCZc6umlxYLpL0kYDA==", - "license": "Apache-2.0" + "version": "2.2.263", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.263.tgz", + "integrity": "sha512-X9JvcJhYcb7PHs8R7m4zMablO5C9PGb/hYfLnxds9h/rKJu6l7MiXE/SabCibuehxPnuO/vk+sVVJiUWrccarQ==" }, "node_modules/@aws-cdk/asset-node-proxy-agent-v6": { "version": "2.1.0", @@ -57,6 +50,41 @@ "integrity": "sha512-7bY3J8GCVxLupn/kNmpPc5VJz8grx+4RKfnnJiO1LG+uxkZfANZG3RMHhE+qQxxwkyQ9/MfPtTpf748UhR425A==", "license": "Apache-2.0" }, + "node_modules/@aws-cdk/cloud-assembly-schema": { + "version": "50.4.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/cloud-assembly-schema/-/cloud-assembly-schema-50.4.0.tgz", + "integrity": "sha512-9Cplwc5C+SNe3hMfqZET7gXeM68tiH2ytQytCi+zz31Bn7O3GAgAnC2dYe+HWnZAgVH788ZkkBwnYXkeqx7v4g==", + "bundleDependencies": [ + "jsonschema", + "semver" + ], + "dependencies": { + "jsonschema": "~1.4.1", + "semver": "^7.7.3" + }, + "engines": { + "node": ">= 18.0.0" + } + }, + "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/jsonschema": { + "version": "1.4.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/semver": { + "version": "7.7.3", + "inBundle": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/@babel/code-frame": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", @@ -1230,11 +1258,12 @@ } }, "node_modules/aws-cdk-lib": { - "version": "2.92.0", - "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.92.0.tgz", - "integrity": "sha512-J+SUFSnOt9u2GbY5QIABgjGNiw8bL/v0S3zsPhhO1dVwK+G7oE+bhLcAi3iILrw2sIpirNWH9K3W0by9K+cyMw==", + "version": "2.240.0", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.240.0.tgz", + "integrity": "sha512-3dXmUnPB5kK0VgrNHOlV3jiQM4Dungukk/CV91nclO2lgNcrGyigauJdzmz9sOmI1gbKJJ2SRAotaXityzZMRw==", "bundleDependencies": [ "@balena/dockerignore", + "@aws-cdk/cloud-assembly-api", "case", "fs-extra", "ignore", @@ -1243,29 +1272,69 @@ "punycode", "semver", "table", - "yaml" + "yaml", + "mime-types" ], - "license": "Apache-2.0", "dependencies": { - "@aws-cdk/asset-awscli-v1": "^2.2.200", - "@aws-cdk/asset-kubectl-v20": "^2.1.2", - "@aws-cdk/asset-node-proxy-agent-v6": "^2.0.1", + "@aws-cdk/asset-awscli-v1": "2.2.263", + "@aws-cdk/asset-node-proxy-agent-v6": "^2.1.0", + "@aws-cdk/cloud-assembly-api": "^2.0.1", + "@aws-cdk/cloud-assembly-schema": "^50.3.0", "@balena/dockerignore": "^1.0.2", "case": "1.6.3", - "fs-extra": "^11.1.1", - "ignore": "^5.2.4", - "jsonschema": "^1.4.1", - "minimatch": "^3.1.2", - "punycode": "^2.3.0", - "semver": "^7.5.4", - "table": "^6.8.1", + "fs-extra": "^11.3.3", + "ignore": "^5.3.2", + "jsonschema": "^1.5.0", + "mime-types": "^2.1.35", + "minimatch": "^10.2.1", + "punycode": "^2.3.1", + "semver": "^7.7.4", + "table": "^6.9.0", "yaml": "1.10.2" }, "engines": { - "node": ">= 14.15.0" + "node": ">= 18.0.0" }, "peerDependencies": { - "constructs": "^10.0.0" + "constructs": "^10.5.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/@aws-cdk/cloud-assembly-api": { + "version": "2.0.1", + "bundleDependencies": [ + "jsonschema", + "semver" + ], + "inBundle": true, + "license": "Apache-2.0", + "dependencies": { + "jsonschema": "~1.4.1", + "semver": "^7.7.3" + }, + "engines": { + "node": ">= 18.0.0" + }, + "peerDependencies": { + "@aws-cdk/cloud-assembly-schema": ">=50.3.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/@aws-cdk/cloud-assembly-api/node_modules/jsonschema": { + "version": "1.4.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/aws-cdk-lib/node_modules/@aws-cdk/cloud-assembly-api/node_modules/semver": { + "version": "7.7.3", + "inBundle": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" } }, "node_modules/aws-cdk-lib/node_modules/@balena/dockerignore": { @@ -1274,14 +1343,14 @@ "license": "Apache-2.0" }, "node_modules/aws-cdk-lib/node_modules/ajv": { - "version": "8.12.0", + "version": "8.18.0", "inBundle": true, "license": "MIT", "dependencies": { - "fast-deep-equal": "^3.1.1", + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" + "require-from-string": "^2.0.2" }, "funding": { "type": "github", @@ -1319,17 +1388,22 @@ } }, "node_modules/aws-cdk-lib/node_modules/balanced-match": { - "version": "1.0.2", + "version": "4.0.4", "inBundle": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } }, "node_modules/aws-cdk-lib/node_modules/brace-expansion": { - "version": "1.1.11", + "version": "5.0.3", "inBundle": true, "license": "MIT", "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" } }, "node_modules/aws-cdk-lib/node_modules/case": { @@ -1356,11 +1430,6 @@ "inBundle": true, "license": "MIT" }, - "node_modules/aws-cdk-lib/node_modules/concat-map": { - "version": "0.0.1", - "inBundle": true, - "license": "MIT" - }, "node_modules/aws-cdk-lib/node_modules/emoji-regex": { "version": "8.0.0", "inBundle": true, @@ -1371,8 +1440,23 @@ "inBundle": true, "license": "MIT" }, + "node_modules/aws-cdk-lib/node_modules/fast-uri": { + "version": "3.1.0", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "inBundle": true, + "license": "BSD-3-Clause" + }, "node_modules/aws-cdk-lib/node_modules/fs-extra": { - "version": "11.1.1", + "version": "11.3.3", "inBundle": true, "license": "MIT", "dependencies": { @@ -1390,7 +1474,7 @@ "license": "ISC" }, "node_modules/aws-cdk-lib/node_modules/ignore": { - "version": "5.2.4", + "version": "5.3.2", "inBundle": true, "license": "MIT", "engines": { @@ -1411,7 +1495,7 @@ "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/jsonfile": { - "version": "6.1.0", + "version": "6.2.0", "inBundle": true, "license": "MIT", "dependencies": { @@ -1422,7 +1506,7 @@ } }, "node_modules/aws-cdk-lib/node_modules/jsonschema": { - "version": "1.4.1", + "version": "1.5.0", "inBundle": true, "license": "MIT", "engines": { @@ -1434,30 +1518,41 @@ "inBundle": true, "license": "MIT" }, - "node_modules/aws-cdk-lib/node_modules/lru-cache": { - "version": "6.0.0", + "node_modules/aws-cdk-lib/node_modules/mime-db": { + "version": "1.52.0", "inBundle": true, - "license": "ISC", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/aws-cdk-lib/node_modules/mime-types": { + "version": "2.1.35", + "inBundle": true, + "license": "MIT", "dependencies": { - "yallist": "^4.0.0" + "mime-db": "1.52.0" }, "engines": { - "node": ">=10" + "node": ">= 0.6" } }, "node_modules/aws-cdk-lib/node_modules/minimatch": { - "version": "3.1.2", + "version": "10.2.2", "inBundle": true, - "license": "ISC", + "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^1.1.7" + "brace-expansion": "^5.0.2" }, "engines": { - "node": "*" + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/aws-cdk-lib/node_modules/punycode": { - "version": "2.3.0", + "version": "2.3.1", "inBundle": true, "license": "MIT", "engines": { @@ -1473,12 +1568,9 @@ } }, "node_modules/aws-cdk-lib/node_modules/semver": { - "version": "7.5.4", + "version": "7.7.4", "inBundle": true, "license": "ISC", - "dependencies": { - "lru-cache": "^6.0.0" - }, "bin": { "semver": "bin/semver.js" }, @@ -1527,7 +1619,7 @@ } }, "node_modules/aws-cdk-lib/node_modules/table": { - "version": "6.8.1", + "version": "6.9.0", "inBundle": true, "license": "BSD-3-Clause", "dependencies": { @@ -1542,26 +1634,13 @@ } }, "node_modules/aws-cdk-lib/node_modules/universalify": { - "version": "2.0.0", + "version": "2.0.1", "inBundle": true, "license": "MIT", "engines": { "node": ">= 10.0.0" } }, - "node_modules/aws-cdk-lib/node_modules/uri-js": { - "version": "4.4.1", - "inBundle": true, - "license": "BSD-2-Clause", - "dependencies": { - "punycode": "^2.1.0" - } - }, - "node_modules/aws-cdk-lib/node_modules/yallist": { - "version": "4.0.0", - "inBundle": true, - "license": "ISC" - }, "node_modules/aws-cdk-lib/node_modules/yaml": { "version": "1.10.2", "inBundle": true, @@ -1690,12 +1769,14 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, "license": "MIT" }, "node_modules/brace-expansion": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -1925,13 +2006,13 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, "license": "MIT" }, "node_modules/constructs": { - "version": "10.4.2", - "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.4.2.tgz", - "integrity": "sha512-wsNxBlAott2qg8Zv87q3eYZYgheb9lchtBfjHzzLHtXbttwSrHPs1NNQbBrmbb1YZvYg2+Vh0Dor76w4mFxJkA==", - "license": "Apache-2.0" + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.5.1.tgz", + "integrity": "sha512-f/TfFXiS3G/yVIXDjOQn9oTlyu9Wo7Fxyjj7lb8r92iO81jR2uST+9MstxZTmDGx/CgIbxCXkFXgupnLTNxQZg==" }, "node_modules/convert-source-map": { "version": "2.0.0", @@ -2038,11 +2119,10 @@ } }, "node_modules/diff": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", - "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.4.tgz", + "integrity": "sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ==", "dev": true, - "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" } @@ -3209,11 +3289,10 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", "dev": true, - "license": "MIT", "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" @@ -3390,10 +3469,10 @@ } }, "node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "license": "ISC", + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.3.tgz", + "integrity": "sha512-M2GCs7Vk83NxkUyQV1bkABc4yxgz9kILhHImZiBPAZ9ybuvCb0/H7lEl5XvIg3g+9d4eNotkZA5IWwYl0tibaA==", + "dev": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -3783,6 +3862,7 @@ "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" diff --git a/cdk/package.json b/cdk/package.json index f1e769db..7cc118ae 100644 --- a/cdk/package.json +++ b/cdk/package.json @@ -13,14 +13,14 @@ "devDependencies": { "@types/jest": "^29.5.3", "@types/node": "20.4.10", + "aws-cdk": "2.92.0", "jest": "^29.6.2", "ts-jest": "^29.1.1", - "aws-cdk": "2.92.0", "ts-node": "^10.9.1", "typescript": "~5.1.6" }, "dependencies": { - "aws-cdk-lib": "2.92.0", + "aws-cdk-lib": "^2.240.0", "constructs": "^10.0.0", "source-map-support": "^0.5.21" } diff --git a/pyproject.toml b/pyproject.toml index 5100857a..a5ab41ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ test = [ dev = [ "black>=24.3.0,<27.0.0", "ruff>=0.3.0", + "boto3-stubs~=1.42.49", ] [build-system] @@ -59,3 +60,4 @@ known-first-party = ["s3_encryption"] [tool.ruff.lint.per-file-ignores] "test/**/*.py" = ["D100", "D101", "D102", "D103", "D104", "E501"] +"src/s3_encryption/pipelines.py" = ["E501"] diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 064096bb..a3558195 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -9,6 +9,7 @@ from botocore.response import StreamingBody from .exceptions import S3EncryptionClientError +from .instruction_file import parse_instruction_file from .materials.crypto_materials_manager import ( AbstractCryptoMaterialsManager, DefaultCryptoMaterialsManager, @@ -25,6 +26,16 @@ class S3EncryptionClientConfig: keyring: AbstractKeyring cmm: AbstractCryptoMaterialsManager = field() + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=implementation + ##% The S3EC SHOULD support providing a custom Instruction File suffix + ##% on GetObject requests, regardless of whether or not re-encryption is supported. + + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=implementation + ##% The default Instruction File behavior uses the same S3 object key + ##% as its associated object suffixed with ".instruction". + instruction_file_suffix: str = field(default=".instruction") @cmm.default def _default_cmm_for_keyring(self): @@ -56,6 +67,11 @@ def on_put_object_before_call(self, params, **kwargs): params: Dictionary of parameters for the PutObject call (after serialization) **kwargs: Additional event arguments """ + if getattr(self._context, "instruction_file_mode", False): + raise S3EncryptionClientError( + "Instruction file mode is exclusively for reading instruction files " + "and not supported in put_object!" + ) # At this point, boto3 has already serialized the Body # Extract the serialized body from the request body = params.get("body") @@ -101,6 +117,11 @@ def on_get_object_after_call(self, parsed, **kwargs): parsed: Dictionary containing the parsed response **kwargs: Additional event arguments (includes 'params' with request parameters) """ + # Check if plaintext mode is enabled via thread-local flag + if getattr(self._context, "instruction_file_mode", False): + self.process_instruction_file(parsed) + return + # Get encryption context from thread-local storage (set by get_object wrapper) encryption_context = getattr(self._context, "encryption_context", None) @@ -114,12 +135,48 @@ def on_get_object_after_call(self, parsed, **kwargs): } # Create a pipeline and decrypt the data - pipeline = GetEncryptedObjectPipeline(self.config.cmm) - decrypted_data = pipeline.decrypt(response, encryption_context) + pipeline = GetEncryptedObjectPipeline( + self.config.cmm, + s3_client=getattr(self._context, "s3_client", None), + ) + decrypted_data = pipeline.decrypt( + response, + encryption_context, + bucket=getattr(self._context, "bucket", None), + key=getattr(self._context, "key", None), + instruction_suffix=self.config.instruction_file_suffix, + ) - # Replace body with decrypted data + # Create a new streaming body with the decrypted data stream = io.BytesIO(decrypted_data) streaming_body = StreamingBody(stream, len(decrypted_data)) + + # Replace body with decrypted data + parsed["Body"] = streaming_body + + def process_instruction_file(self, parsed): + """Process instruction file in plaintext mode. + + Validates the instruction file marker, parses the JSON body, + and updates the response metadata with parsed content. + + Args: + parsed: Dictionary containing the parsed response + """ + instruction_key = getattr(self._context, "key", None) + + # In plaintext mode, parse instruction file and append to metadata + existing_metadata = parsed.get("Metadata", {}) + instruction_data = parsed.get("Body").read() + instruction_metadata = parse_instruction_file(instruction_data, instruction_key) + + # Append parsed instruction file content to existing metadata + existing_metadata.update(instruction_metadata) + parsed["Metadata"] = existing_metadata + + # Clear the body since instruction files shouldn't return body content + stream = io.BytesIO(b"") + streaming_body = StreamingBody(stream, 0) parsed["Body"] = streaming_body @@ -143,6 +200,9 @@ def __attrs_post_init__(self): # Create the plugin object.__setattr__(self, "_plugin", S3EncryptionClientPlugin(self.config)) + # Expose plugin context on wrapped client for instruction file fetching + self.wrapped_s3_client._s3ec_plugin_context = self._plugin._context + # Register event handlers using boto3's event system event_system = self.wrapped_s3_client.meta.events event_system.register("before-call.s3.PutObject", self._plugin.on_put_object_before_call) @@ -207,6 +267,11 @@ def get_object(self, **kwargs): # Store encryption context in thread-local storage for the event handler self._plugin._context.encryption_context = encryption_context + # Store wrapped client in thread-local storage for + # the event handler to fetch instruction files + self._plugin._context.s3_client = self.wrapped_s3_client + self._plugin._context.bucket = kwargs.get("Bucket") + self._plugin._context.key = kwargs.get("Key") try: return self.wrapped_s3_client.get_object(**kwargs) @@ -217,6 +282,9 @@ def get_object(self, **kwargs): # Wrap any unexpected errors during decryption raise S3EncryptionClientError(f"Failed to decrypt object: {str(e)}") from e finally: - # Clean up thread-local storage - if hasattr(self._plugin._context, "encryption_context"): - delattr(self._plugin._context, "encryption_context") + # Clean up thread-local storage; + # do not clean up the client as it is not thread local only + attrs = ["encryption_context", "Bucket", "Key"] + for attr in attrs: + if hasattr(self._plugin._context, attr): + delattr(self._plugin._context, attr) diff --git a/src/s3_encryption/instruction_file.py b/src/s3_encryption/instruction_file.py new file mode 100644 index 00000000..351c4b15 --- /dev/null +++ b/src/s3_encryption/instruction_file.py @@ -0,0 +1,117 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Instruction file handling for S3 Encryption Client. + +This module provides utilities for fetching and parsing instruction files +that contain encryption metadata for S3 objects. +""" + +import json +from typing import Any + +from .exceptions import S3EncryptionClientError +from .metadata import VALID_S3EC_METADATA_KEYS + + +def parse_instruction_file(instruction_data: bytes, key: str) -> dict[str, Any]: + """Parse and validate instruction file data. + + This function strictly validates that: + 1. The instruction file body is valid JSON + 2. The JSON contains only S3 Encryption Client metadata keys + + Args: + instruction_data: Raw bytes from instruction file body + key: Instruction file key (for error messages) + + Returns: + dict: Parsed JSON metadata from instruction file + + Raises: + S3EncryptionClientError: If the instruction file is not valid JSON + or contains non-S3EC metadata keys + """ + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=implementation + ##% The content metadata stored in the Instruction File MUST be serialized to a JSON string. + + # Validate JSON format + try: + metadata = json.loads(instruction_data) + except json.JSONDecodeError as e: + raise S3EncryptionClientError(f"Instruction file is not valid JSON: {key}") from e + + # Validate that it's a dictionary + if not isinstance(metadata, dict): + raise S3EncryptionClientError( + f"Instruction file must contain a JSON object, " f"got {type(metadata).__name__}: {key}" + ) + + # Validate that all keys are S3EC metadata keys + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=implementation + ##% The serialized JSON string MUST be the only contents of the Instruction File. + invalid_keys = set(metadata.keys()) - VALID_S3EC_METADATA_KEYS + if invalid_keys: + raise S3EncryptionClientError( + f"Instruction file contains invalid keys: {invalid_keys} in {key}" + ) + + return metadata + + +def fetch_instruction_file(s3_client, bucket: str, key: str) -> dict[str, Any]: + """Fetch and parse an instruction file from S3. + + This function: + 1. Fetches the instruction file in plaintext mode + 2. Returns the parsed metadata from the response Metadata field + + S3EncryptionClientPlugin's event handler (on_get_object_after_call) handles: + - Parsing and validating the instruction file content + - Placing parsed metadata in response["Metadata"] + + Args: + s3_client: Boto3 S3 client to use for fetching + bucket: S3 bucket name + key: S3 object key + Returns: + dict: Parsed JSON metadata from instruction file + + Raises: + S3EncryptionClientError: If the instruction file is not valid JSON, + or contains non-S3EC metadata keys + """ + # Set plaintext mode flag in thread-local context before calling get_object + # This will be checked by the event handler to skip decryption + if hasattr(s3_client, "_s3ec_plugin_context"): + s3_client._s3ec_plugin_context.instruction_file_mode = True + s3_client._s3ec_plugin_context.key = key + else: + raise S3EncryptionClientError( + f"Could not fetch instruction file without " + f"the S3 Encryption Client Plugin installed. Instruction key: {key}" + ) + + try: + response = s3_client.get_object(Bucket=bucket, Key=key) + finally: + # Clear the flags after the call + if hasattr(s3_client, "_s3ec_plugin_context"): + s3_client._s3ec_plugin_context.instruction_file_mode = False + + # In plaintext mode, the event handler places parsed metadata in Metadata field + metadata = response.get("Metadata", {}) + + # Verify metadata is not empty + if not metadata: + raise S3EncryptionClientError(f"Instruction file returned empty metadata: {key}") + + # Verify metadata contains at least one S3EC key + has_s3ec_key = any(key in VALID_S3EC_METADATA_KEYS for key in metadata) + if not has_s3ec_key: + raise S3EncryptionClientError( + f"Instruction file metadata does not contain any S3EC keys: {key}" + ) + + return metadata diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py index f42feadb..5c8bbda3 100644 --- a/src/s3_encryption/metadata.py +++ b/src/s3_encryption/metadata.py @@ -48,6 +48,15 @@ class ObjectMetadata: # Marker for instruction files instruction_file: str | None = field(default=None) + # V3 format fields (compressed) + content_cipher_v3: str | None = field(default=None) + encrypted_data_key_v3: str | None = field(default=None) + mat_desc_v3: str | None = field(default=None) + encryption_context_v3: str | None = field(default=None) + encrypted_data_key_algorithm_v3: str | None = field(default=None) + key_commitment_v3: str | None = field(default=None) + message_id_v3: str | None = field(default=None) + # Constants for metadata keys ENCRYPTED_DATA_KEY_V1 = "x-amz-key" ENCRYPTED_DATA_KEY_V2 = "x-amz-key-v2" @@ -58,6 +67,15 @@ class ObjectMetadata: CONTENT_CIPHER_TAG_LENGTH = "x-amz-tag-len" INSTRUCTION_FILE = "x-amz-crypto-instr-file" + # V3 format constants (compressed) + CONTENT_CIPHER_V3 = "x-amz-c" + ENCRYPTED_DATA_KEY_V3 = "x-amz-3" + MAT_DESC_V3 = "x-amz-m" + ENCRYPTION_CONTEXT_V3 = "x-amz-t" + ENCRYPTED_DATA_KEY_ALGORITHM_V3 = "x-amz-w" + KEY_COMMITMENT_V3 = "x-amz-d" + MESSAGE_ID_V3 = "x-amz-i" + @classmethod def from_dict(cls, metadata_dict: dict[str, Any]) -> "ObjectMetadata": """Create an ObjectMetadata instance from a dictionary. @@ -84,6 +102,13 @@ def from_dict(cls, metadata_dict: dict[str, Any]) -> "ObjectMetadata": content_cipher=metadata_dict.get(cls.CONTENT_CIPHER), content_cipher_tag_length=metadata_dict.get(cls.CONTENT_CIPHER_TAG_LENGTH), instruction_file=metadata_dict.get(cls.INSTRUCTION_FILE), + content_cipher_v3=metadata_dict.get(cls.CONTENT_CIPHER_V3), + encrypted_data_key_v3=metadata_dict.get(cls.ENCRYPTED_DATA_KEY_V3), + mat_desc_v3=metadata_dict.get(cls.MAT_DESC_V3), + encryption_context_v3=metadata_dict.get(cls.ENCRYPTION_CONTEXT_V3), + encrypted_data_key_algorithm_v3=metadata_dict.get(cls.ENCRYPTED_DATA_KEY_ALGORITHM_V3), + key_commitment_v3=metadata_dict.get(cls.KEY_COMMITMENT_V3), + message_id_v3=metadata_dict.get(cls.MESSAGE_ID_V3), ) def to_dict(self) -> dict[str, str]: @@ -118,4 +143,145 @@ def to_dict(self) -> dict[str, str]: if self.instruction_file is not None: result[self.INSTRUCTION_FILE] = self.instruction_file + if self.content_cipher_v3 is not None: + result[self.CONTENT_CIPHER_V3] = self.content_cipher_v3 + + if self.encrypted_data_key_v3 is not None: + result[self.ENCRYPTED_DATA_KEY_V3] = self.encrypted_data_key_v3 + + if self.mat_desc_v3 is not None: + result[self.MAT_DESC_V3] = self.mat_desc_v3 + + if self.encryption_context_v3 is not None: + result[self.ENCRYPTION_CONTEXT_V3] = self.encryption_context_v3 + + if self.encrypted_data_key_algorithm_v3 is not None: + result[self.ENCRYPTED_DATA_KEY_ALGORITHM_V3] = self.encrypted_data_key_algorithm_v3 + + if self.key_commitment_v3 is not None: + result[self.KEY_COMMITMENT_V3] = self.key_commitment_v3 + + if self.message_id_v3 is not None: + result[self.MESSAGE_ID_V3] = self.message_id_v3 + return result + + def is_v1_format(self) -> bool: + """Check if metadata is in V1 format. + + Returns: + bool: True if metadata contains V1 keys and excludes V2/V3 keys + """ + return ( + self.content_iv is not None + and self.encrypted_data_key_context is not None + and self.encrypted_data_key_v1 is not None + and self.encrypted_data_key_v2 is None + ) + + def is_v2_format(self) -> bool: + """Check if metadata is in V2 format. + + Returns: + bool: True if metadata contains V2 keys and excludes V1/V3 keys + """ + return ( + self.content_cipher is not None + and self.content_iv is not None + and self.encrypted_data_key_algorithm is not None + and self.encrypted_data_key_v2 is not None + and self.encrypted_data_key_v1 is None + ) + + def is_v3_format(self) -> bool: + """Check if metadata is in V3 format. + + Returns: + bool: True if metadata contains V3 keys and excludes V1/V2 keys + """ + return ( + self.content_cipher_v3 is not None + and self.encrypted_data_key_algorithm_v3 is not None + and self.key_commitment_v3 is not None + and self.message_id_v3 is not None + and self.encrypted_data_key_v3 is not None + and self.encrypted_data_key_v2 is None + and self.encrypted_data_key_v1 is None + ) + + def has_exclusive_key_collision(self) -> bool: + """Check if metadata has multiple exclusive version keys. + + Returns: + bool: True if more than one version key (V1, V2, V3) is present + """ + has_v1_key = self.encrypted_data_key_v1 is not None + has_v2_key = self.encrypted_data_key_v2 is not None + has_v3_key = self.encrypted_data_key_v3 is not None + + exclusive_key_count = sum([has_v1_key, has_v2_key, has_v3_key]) + return exclusive_key_count > 1 + + def is_v3_in_object_metadata(self) -> bool: + """Check if V3 content keys are in object metadata (without encrypted data key). + + ##= specification/s3-encryption/data-format/metadata-strategy.md#v3-instruction-files + ##= type=implementation + ##% In the V3 message format, only the content metadata related to + ##% the encrypted data is stored in the Instruction File. + ##% In the V3 message format, the content metadata related to + ##% the encrypted content is stored in the Object Metadata. + + Returns: + bool: True if V3 content keys present but no encrypted data key + """ + return ( + self.content_cipher_v3 is not None + and self.key_commitment_v3 is not None + and self.message_id_v3 is not None + and self.encrypted_data_key_v3 is None + ) + + ##= specification/s3-encryption/data-format/content-metadata.md#determining-s3ec-object-status + ##= type=implementation + ##% If the object matches none of the V1/V2/V3 formats, + ##% the S3EC MUST attempt to get the instruction file. + def should_use_instruction_file(self) -> bool: + """Check if instruction file should be used for decryption. + + Returns: + bool: True if instruction file should be fetched + """ + # V3 with content keys but no encrypted data key -> instruction file + if self.is_v3_in_object_metadata(): + return True + + # No version keys at all -> try instruction file for V1/V2 + has_any_key = ( + self.encrypted_data_key_v1 is not None + or self.encrypted_data_key_v2 is not None + or self.encrypted_data_key_v3 is not None + ) + return not has_any_key + + +# Valid S3 Encryption Client metadata keys +VALID_S3EC_METADATA_KEYS = { + # V1/V2 format keys + "x-amz-key", + "x-amz-key-v2", + "x-amz-wrap-alg", + "x-amz-matdesc", + "x-amz-iv", + "x-amz-cek-alg", + "x-amz-tag-len", + "x-amz-crypto-instr-file", + # V3 format keys (compressed) + "x-amz-c", + "x-amz-3", + "x-amz-m", + "x-amz-t", + "x-amz-w", + "x-amz-d", + "x-amz-i", +} diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 3a83a359..02a5a9c9 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -12,6 +12,8 @@ from attrs import define, field from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from .exceptions import S3EncryptionClientError +from .instruction_file import fetch_instruction_file from .materials.crypto_materials_manager import AbstractCryptoMaterialsManager from .materials.encrypted_data_key import EncryptedDataKey from .materials.materials import DecryptionMaterials, EncryptionMaterials @@ -90,13 +92,24 @@ class GetEncryptedObjectPipeline: """ cmm: AbstractCryptoMaterialsManager = field() - - def decrypt(self, response, encryption_context=None): + s3_client: object = field(default=None) + + def decrypt( + self, + response, + encryption_context=None, + bucket=None, + key=None, + instruction_suffix=".instruction", + ): """Decrypt the data after it is retrieved from S3. Args: response (dict): The response from S3 containing the encrypted data and metadata encryption_context (dict, optional): Additional context for decryption + bucket (str, optional): S3 bucket name (required for instruction file) + key (str, optional): S3 object key (required for instruction file) + instruction_suffix(str, optional): suffix for instruction file; defaults to ".instruction". Returns: bytes: The decrypted data @@ -111,44 +124,45 @@ def decrypt(self, response, encryption_context=None): if encryption_context is None: encryption_context = {} - iv_b64 = metadata.content_iv - edk_b64 = metadata.encrypted_data_key_v2 - - # TODO: probably move this to ObjectMetadata - iv_bytes = base64.b64decode(iv_b64) - - # Create a list of encrypted data keys to try - encrypted_data_keys = [] - # Create an instance of EncryptedDataKey - if edk_b64: - edk_bytes = base64.b64decode(edk_b64) - encrypted_data_key = EncryptedDataKey( - key_provider_id=b"S3Keyring", - key_provider_info=metadata.encrypted_data_key_algorithm, - encrypted_data_key=edk_bytes, - ) - encrypted_data_keys.append(encrypted_data_key) - - # Also check for legacy encrypted data key (v1) if available - if metadata.encrypted_data_key_v1: - legacy_edk_bytes = base64.b64decode(metadata.encrypted_data_key_v1) - legacy_encrypted_data_key = EncryptedDataKey( - key_provider_id=b"S3Keyring", - key_provider_info=metadata.encrypted_data_key_algorithm, - encrypted_data_key=legacy_edk_bytes, + # Check if we need to fetch instruction file + if metadata.should_use_instruction_file(): + + if self.s3_client is None: + raise S3EncryptionClientError("s3_client required to fetch instruction file") + if bucket is None or key is None: + raise S3EncryptionClientError("Bucket and key required to fetch instruction file") + + instruction_key = key + instruction_suffix + instruction_metadata = fetch_instruction_file(self.s3_client, bucket, instruction_key) + instruction_metadata.update(encryption_metadata) + metadata = ObjectMetadata.from_dict(instruction_metadata) + ##= specification/s3-encryption/data-format/metadata-strategy.md#v1-v2-instruction-files + ##= type=implementation + ##% In the V1/V2 message format, all of the content metadata + ##% MUST be stored in the Instruction File. + if metadata.is_v1_format() or metadata.is_v2_format(): + object_metadata = ObjectMetadata.from_dict(encryption_metadata) + if not ( + object_metadata.content_cipher is None + and object_metadata.content_iv is None + and object_metadata.encrypted_data_key_algorithm is None + ): + raise S3EncryptionClientError( + "Content metadata found in object metadata for V1 or V2 message format " + "BUT Instruction File is being used. This is an illegal combination. " + f"bucket: {bucket}\n key:{key}\n instruction_file:{instruction_key}" + ) + # Determine which format we're dealing with and get decryption materials + if metadata.is_v1_format(): + dec_materials = self._decrypt_v1(metadata, encryption_context) + elif metadata.is_v2_format(): + dec_materials = self._decrypt_v2(metadata, encryption_context) + elif metadata.is_v3_format(): + dec_materials = self._decrypt_v3(metadata, encryption_context) + else: + raise S3EncryptionClientError( + "Unable to determine S3 Encryption Client message format." ) - encrypted_data_keys.append(legacy_encrypted_data_key) - - # Create a DecryptionMaterials instance - dec_materials = DecryptionMaterials( - iv=iv_bytes, - encrypted_data_keys=encrypted_data_keys, - encryption_context_stored=metadata.encrypted_data_key_context or {}, - encryption_context_from_request=encryption_context or {}, - ) - - # Get decryption materials from the crypto materials manager - dec_materials = self.cmm.decrypt_materials(dec_materials) ##= specification/s3-encryption/decryption.md#cbc-decryption ##= type=TODO @@ -157,6 +171,51 @@ def decrypt(self, response, encryption_context=None): ##% the S3EC MUST throw an error which details that client was ##% not configured to decrypt objects with ALG_AES_256_CBC_IV16_NO_KDF. + # Perform decryption aesgcm = AESGCM(dec_materials.plaintext_data_key) + return aesgcm.decrypt(nonce=dec_materials.iv, data=encrypted_data, associated_data=None) + + def _decrypt_v2(self, metadata, encryption_context) -> DecryptionMaterials: + """Prepare V2 decryption materials.""" + iv_bytes = base64.b64decode(metadata.content_iv) + edk_bytes = base64.b64decode(metadata.encrypted_data_key_v2) + + encrypted_data_key = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info=metadata.encrypted_data_key_algorithm, + encrypted_data_key=edk_bytes, + ) + + dec_materials = DecryptionMaterials( + iv=iv_bytes, + encrypted_data_keys=[encrypted_data_key], + encryption_context_stored=metadata.encrypted_data_key_context or {}, + encryption_context_from_request=encryption_context, + ) + + return self.cmm.decrypt_materials(dec_materials) + + def _decrypt_v1(self, metadata, encryption_context) -> DecryptionMaterials: + """Prepare V1 decryption materials.""" + iv_bytes = base64.b64decode(metadata.content_iv) + edk_bytes = base64.b64decode(metadata.encrypted_data_key_v1) + + encrypted_data_key = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info=metadata.encrypted_data_key_algorithm, + encrypted_data_key=edk_bytes, + ) + + dec_materials = DecryptionMaterials( + iv=iv_bytes, + encrypted_data_keys=[encrypted_data_key], + encryption_context_stored=metadata.encrypted_data_key_context or {}, + encryption_context_from_request=encryption_context, + ) + + return self.cmm.decrypt_materials(dec_materials) - return aesgcm.decrypt(nonce=iv_bytes, data=encrypted_data, associated_data=None) + def _decrypt_v3(self, metadata, encryption_context) -> DecryptionMaterials: + """Prepare V3 decryption materials.""" + # TODO: Implement V3 decryption + raise NotImplementedError("V3 decryption not yet implemented") diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py new file mode 100644 index 00000000..467ddc7a --- /dev/null +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -0,0 +1,151 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import os + +import boto3 +import pytest + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.materials.kms_keyring import KmsKeyring + +# Static test objects bucket +bucket = os.environ.get("CI_S3_STATIC_TEST_BUCKET", "s3ec-static-test-objects") +region = os.environ.get("CI_AWS_REGION", "us-west-2") +# KMS key used for static test objects (S3ECTestServerKMSKey) +kms_key_id = os.environ.get( + "CI_KMS_KEY_STATIC_TESTS", + "arn:aws:kms:us-west-2:370957321024:key/a3889cd9-99eb-4138-a93a-aea9d52ec2ef", +) + +# Static test object keys created by Java S3EC V4 +TEST_OBJECTS = { + "v1_instruction_file": "static-v1-instruction-file-from-java-v1", + "v2_instruction_file": "static-v2-instruction-file-from-java-v4", + "v3_instruction_file": "static-v3-instruction-file-from-java-v4", + "negative_v2_instruction_file": "NEGATIVE-static-v2-instruction-file-test-from-java-v4", +} + + +# TODO(cbc): enable once CBC decryption is implemented +@pytest.mark.skip(reason="V1 CBC decryption not yet implemented") +def test_decrypt_v1_instruction_file(): + """Test decrypting V1 object with instruction file. + + V1 format uses ALG_AES_256_CBC_IV16_NO_KDF (CBC mode, no key commitment). + Object encrypted by Java S3EC V1 with instruction file enabled. + """ + key = TEST_OBJECTS["v1_instruction_file"] + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id, enable_legacy_wrapping_algorithms=True) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + response = s3ec.get_object(Bucket=bucket, Key=key) + output = response["Body"].read().decode("utf-8") + + assert output == "static-v1-instruction-file-from-java-v1" + print("Success! V1 instruction file decryption completed.") + + +def test_decrypt_v2_instruction_file(): + """Test decrypting V2 object with instruction file. + + V2 format uses ALG_AES_256_GCM_IV12_TAG16_NO_KDF (no key commitment). + Object encrypted by Java S3EC V4 with instruction file enabled. + """ + key = TEST_OBJECTS["v2_instruction_file"] + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + response = s3ec.get_object(Bucket=bucket, Key=key) + output = response["Body"].read().decode("utf-8") + + assert output == "static-v2-instruction-file-from-java-v4" + print("Success! V2 instruction file decryption completed.") + + +# TODO(v3): enable once v3 is implemented +@pytest.mark.skip(reason="V3 decryption not yet implemented") +def test_decrypt_v3_instruction_file(): + """Test decrypting V3 object with instruction file. + + V3 format uses ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY (with key commitment). + Object encrypted by Java S3EC V4 with instruction file enabled. + """ + key = TEST_OBJECTS["v3_instruction_file"] + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + response = s3ec.get_object(Bucket=bucket, Key=key) + output = response["Body"].read().decode("utf-8") + + assert output != "static-v3-instruction-file-from-java-v4" + print("Success! V3 instruction file decryption completed.") + + +def test_decrypt_invalid_instruction_file(): + """Test that decrypting with an invalid instruction file raises an error. + + The NEGATIVE test object has an invalid instruction file that should + cause the S3 Encryption Client to raise an exception during decryption. + """ + from s3_encryption.exceptions import S3EncryptionClientError + + key = TEST_OBJECTS["negative_v2_instruction_file"] + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + with pytest.raises(S3EncryptionClientError) as exc_info: + s3ec.get_object(Bucket=bucket, Key=key) + + print(f"Error message: {exc_info.value}") + + +# TODO(v3): enable once v3 is implemented +@pytest.mark.skip(reason="V3 decryption not yet implemented") +def test_decrypt_v3_instruction_file_custom_suffix(): + """Test decrypting V3 object with a custom instruction file suffix.""" + key = TEST_OBJECTS["v3_instruction_file"] + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring, instruction_file_suffix=".custom-suffix-instruction") + s3ec = S3EncryptionClient(wrapped_client, config) + + response = s3ec.get_object(Bucket=bucket, Key=key) + output = response["Body"].read().decode("utf-8") + + assert output == "static-v3-instruction-file-from-java-v4" + print("Success! V3 custom suffix instruction file decryption completed.") + + +def test_decrypt_v2_instruction_file_custom_suffix(): + """Test decrypting V2 object with a custom instruction file suffix.""" + key = TEST_OBJECTS["v2_instruction_file"] + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring, instruction_file_suffix=".custom-suffix-instruction") + s3ec = S3EncryptionClient(wrapped_client, config) + + response = s3ec.get_object(Bucket=bucket, Key=key) + output = response["Body"].read().decode("utf-8") + + assert output == "static-v2-instruction-file-from-java-v4" + print("Success! V2 custom suffix instruction file decryption completed.") diff --git a/test/test_metadata.py b/test/test_metadata.py index a061c185..ba783bf5 100644 --- a/test/test_metadata.py +++ b/test/test_metadata.py @@ -79,3 +79,160 @@ def test_roundtrip(self): # Verify that the result matches the original assert result_dict == original_dict + + def test_from_dict_v3_fields(self): + # Create a metadata dictionary with V3 fields + metadata_dict = { + "x-amz-c": "02", + "x-amz-3": "encrypted-key-v3", + "x-amz-w": "12", + "x-amz-d": "key-commitment", + "x-amz-i": "message-id", + "x-amz-m": "mat-desc", + "x-amz-t": "encryption-context", + } + + metadata = ObjectMetadata.from_dict(metadata_dict) + + assert metadata.content_cipher_v3 == "02" + assert metadata.encrypted_data_key_v3 == "encrypted-key-v3" + assert metadata.encrypted_data_key_algorithm_v3 == "12" + assert metadata.key_commitment_v3 == "key-commitment" + assert metadata.message_id_v3 == "message-id" + assert metadata.mat_desc_v3 == "mat-desc" + assert metadata.encryption_context_v3 == "encryption-context" + + def test_to_dict_v3_fields(self): + # Create an ObjectMetadata instance with V3 fields + metadata = ObjectMetadata( + content_cipher_v3="02", + encrypted_data_key_v3="encrypted-key-v3", + encrypted_data_key_algorithm_v3="12", + key_commitment_v3="key-commitment", + message_id_v3="message-id", + mat_desc_v3="mat-desc", + encryption_context_v3="encryption-context", + ) + + metadata_dict = metadata.to_dict() + + assert metadata_dict["x-amz-c"] == "02" + assert metadata_dict["x-amz-3"] == "encrypted-key-v3" + assert metadata_dict["x-amz-w"] == "12" + assert metadata_dict["x-amz-d"] == "key-commitment" + assert metadata_dict["x-amz-i"] == "message-id" + assert metadata_dict["x-amz-m"] == "mat-desc" + assert metadata_dict["x-amz-t"] == "encryption-context" + + def test_is_v1_format(self): + metadata = ObjectMetadata( + content_iv="iv", + encrypted_data_key_context={"key": "value"}, + encrypted_data_key_v1="edk-v1", + ) + assert metadata.is_v1_format() is True + + # V2 key present should return False + metadata_v2 = ObjectMetadata( + content_iv="iv", + encrypted_data_key_context={"key": "value"}, + encrypted_data_key_v1="edk-v1", + encrypted_data_key_v2="edk-v2", + ) + assert metadata_v2.is_v1_format() is False + + def test_is_v2_format(self): + metadata = ObjectMetadata( + content_cipher="AES/GCM/NoPadding", + content_iv="iv", + encrypted_data_key_algorithm="kms+context", + encrypted_data_key_v2="edk-v2", + ) + assert metadata.is_v2_format() is True + + # V1 key present should return False + metadata_v1 = ObjectMetadata( + content_cipher="AES/GCM/NoPadding", + content_iv="iv", + encrypted_data_key_algorithm="kms+context", + encrypted_data_key_v2="edk-v2", + encrypted_data_key_v1="edk-v1", + ) + assert metadata_v1.is_v2_format() is False + + def test_is_v3_format(self): + metadata = ObjectMetadata( + content_cipher_v3="02", + encrypted_data_key_algorithm_v3="12", + key_commitment_v3="commitment", + message_id_v3="msg-id", + encrypted_data_key_v3="edk-v3", + ) + assert metadata.is_v3_format() is True + + # V1 or V2 keys present should return False + metadata_v2 = ObjectMetadata( + content_cipher_v3="02", + encrypted_data_key_algorithm_v3="12", + key_commitment_v3="commitment", + message_id_v3="msg-id", + encrypted_data_key_v3="edk-v3", + encrypted_data_key_v2="edk-v2", + ) + assert metadata_v2.is_v3_format() is False + + def test_has_exclusive_key_collision(self): + # No collision - only V2 + metadata_v2 = ObjectMetadata(encrypted_data_key_v2="edk-v2") + assert metadata_v2.has_exclusive_key_collision() is False + + # Collision - V1 and V2 + metadata_collision = ObjectMetadata( + encrypted_data_key_v1="edk-v1", + encrypted_data_key_v2="edk-v2", + ) + assert metadata_collision.has_exclusive_key_collision() is True + + # Collision - all three + metadata_all = ObjectMetadata( + encrypted_data_key_v1="edk-v1", + encrypted_data_key_v2="edk-v2", + encrypted_data_key_v3="edk-v3", + ) + assert metadata_all.has_exclusive_key_collision() is True + + ##= specification/s3-encryption/data-format/content-metadata.md#determining-s3ec-object-status + ##= type=test + ##% If the object matches none of the V1/V2/V3 formats, + ##% the S3EC MUST attempt to get the instruction file. + def test_should_use_instruction_file(self): + # No keys at all -> should use instruction file + metadata_empty = ObjectMetadata() + assert metadata_empty.should_use_instruction_file() is True + + # V3 in object metadata (has content keys but no EDK) -> instruction file + metadata_v3_partial = ObjectMetadata( + content_cipher_v3="02", + encrypted_data_key_algorithm_v3="12", + key_commitment_v3="commitment", + message_id_v3="msg-id", + ) + assert metadata_v3_partial.should_use_instruction_file() is True + + # V1 with EDK -> no instruction file needed + metadata_v1 = ObjectMetadata(encrypted_data_key_v1="edk-v1") + assert metadata_v1.should_use_instruction_file() is False + + # V2 with EDK -> no instruction file needed + metadata_v2 = ObjectMetadata(encrypted_data_key_v2="edk-v2") + assert metadata_v2.should_use_instruction_file() is False + + # V3 with EDK -> no instruction file needed + metadata_v3 = ObjectMetadata( + content_cipher_v3="02", + encrypted_data_key_algorithm_v3="12", + key_commitment_v3="commitment", + message_id_v3="msg-id", + encrypted_data_key_v3="edk-v3", + ) + assert metadata_v3.should_use_instruction_file() is False diff --git a/test/test_pipelines.py b/test/test_pipelines.py new file mode 100644 index 00000000..9f40cd5c --- /dev/null +++ b/test/test_pipelines.py @@ -0,0 +1,241 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import base64 +import json +import os +from io import BytesIO +from unittest.mock import Mock + +import pytest + +from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager +from s3_encryption.materials.keyring import S3Keyring +from s3_encryption.pipelines import GetEncryptedObjectPipeline + + +class TestGetEncryptedObjectPipelineInstructionFile: + ##= specification/s3-encryption/data-format/metadata-strategy.md#v1-v2-instruction-files + ##= type=test + ##% In the V1/V2 message format, all of the content metadata + ##% MUST be stored in the Instruction File. + def test_decrypt_v1_from_instruction_file(self): + """Test decrypting V1 format with instruction file.""" + object_metadata = {"x-amz-meta-x-amz-unencrypted-content-length": "39"} + + # Instruction file contains all V1 metadata + instruction_file_metadata = { + "x-amz-iv": base64.b64encode(os.urandom(16)).decode("utf-8"), + "x-amz-key-v2": base64.b64encode(b"encrypted-key-data").decode("utf-8"), + "x-amz-wrap-alg": "kms", + "x-amz-matdesc": json.dumps({"kms_cmk_id": "test-key-id"}), + "x-amz-cek-alg": "AES/CBC/PKCS5Padding", + "x-amz-crypto-instr-file": "", + } + + # Create mock S3 client + mock_s3_client = Mock() + # Mock returns parsed metadata (simulating event handler behavior) + mock_s3_client.get_object.return_value = { + "Body": BytesIO(b""), # Body is cleared by event handler + "Metadata": instruction_file_metadata, + } + + # Create mock keyring and CMM + mock_keyring = Mock(spec=S3Keyring) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + + # Create pipeline with mocked S3 client + pipeline = GetEncryptedObjectPipeline(cmm, mock_s3_client) + + # Create mock response + mock_response = { + "Body": BytesIO(b"encrypted-test-data"), + "Metadata": object_metadata, + } + + # Mock the keyring to raise an error so we don't actually decrypt + mock_keyring.on_decrypt.side_effect = Exception( + "Keyring called - instruction file was fetched" + ) + + # Should fail when trying to decrypt (proving instruction file was fetched) + with pytest.raises(Exception, match="Keyring called"): + pipeline.decrypt(mock_response, bucket="test-bucket", key="test-key") + + # Verify instruction file was fetched + mock_s3_client.get_object.assert_called_once_with( + Bucket="test-bucket", Key="test-key.instruction" + ) + + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=test + ##% The default Instruction File behavior uses the same S3 object key + ##% as its associated object suffixed with ".instruction". + def test_decrypt_v2_from_instruction_file(self): + """Test decrypting V2 format with instruction file.""" + # V2: Object metadata is empty, all metadata in instruction file + object_metadata = {} + + # Instruction file contains all V2 metadata + instruction_file_metadata = { + "x-amz-iv": base64.b64encode(os.urandom(12)).decode("utf-8"), + "x-amz-key-v2": base64.b64encode(b"encrypted-key-data").decode("utf-8"), + "x-amz-wrap-alg": "kms+context", + "x-amz-matdesc": json.dumps({"kms_cmk_id": "test-key-id"}), + "x-amz-cek-alg": "AES/GCM/NoPadding", + "x-amz-tag-len": "128", + "x-amz-crypto-instr-file": "", + } + + # Create mock S3 client + mock_s3_client = Mock() + # Mock returns parsed metadata (simulating event handler behavior) + mock_s3_client.get_object.return_value = { + "Body": BytesIO(b""), # Body is cleared by event handler + "Metadata": instruction_file_metadata, + } + + # Create mock keyring and CMM + mock_keyring = Mock(spec=S3Keyring) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + + # Create pipeline with mocked S3 client + pipeline = GetEncryptedObjectPipeline(cmm, mock_s3_client) + + # Create mock response + mock_response = { + "Body": BytesIO(b"encrypted-test-data"), + "Metadata": object_metadata, + } + + # Mock the keyring to raise an error so we don't actually decrypt + mock_keyring.on_decrypt.side_effect = Exception( + "Keyring called - instruction file was fetched" + ) + + # Should fail when trying to decrypt (proving instruction file was fetched) + with pytest.raises(Exception, match="Keyring called"): + pipeline.decrypt(mock_response, bucket="test-bucket", key="test-key") + + # Verify instruction file was fetched + mock_s3_client.get_object.assert_called_once_with( + Bucket="test-bucket", Key="test-key.instruction" + ) + + ##= specification/s3-encryption/data-format/metadata-strategy.md#v3-instruction-files + ##= type=test + ##% In the V3 message format, only the content metadata related to + ##% the encrypted data is stored in the Instruction File. + def test_decrypt_v3_from_instruction_file(self): + """Test decrypting V3 format with instruction file.""" + # Object metadata contains V3 content keys only + object_metadata = { + "x-amz-c": "115", # Compressed algorithm suite + "x-amz-d": base64.b64encode(b"key-commitment-data").decode("utf-8"), + "x-amz-i": base64.b64encode(b"test-message-id").decode("utf-8"), + } + + # Instruction file contains encrypted data key and wrapping algorithm + instruction_file_metadata = { + "x-amz-3": base64.b64encode(b"encrypted-key-data").decode("utf-8"), + "x-amz-w": "02", # AES/GCM + "x-amz-m": json.dumps({"test-instruction": "material-desc-instruction"}), + "x-amz-crypto-instr-file": "", + } + + # Create mock S3 client + mock_s3_client = Mock() + # Mock returns parsed metadata (simulating event handler behavior) + mock_s3_client.get_object.return_value = { + "Body": BytesIO(b""), # Body is cleared by event handler + "Metadata": instruction_file_metadata, + } + + # Create mock keyring and CMM + mock_keyring = Mock(spec=S3Keyring) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + + # Create pipeline with mocked S3 client + pipeline = GetEncryptedObjectPipeline(cmm, mock_s3_client) + + # Create mock response with encrypted data + iv = os.urandom(12) + encrypted_data = b"encrypted-test-data" + + mock_response = { + "Body": BytesIO(encrypted_data), + "Metadata": object_metadata, + } + + # Mock the keyring to return decryption materials + from s3_encryption.materials.materials import DecryptionMaterials + + plaintext_data_key = os.urandom(32) + + mock_dec_materials = DecryptionMaterials( + iv=iv, + encrypted_data_keys=[], + encryption_context_stored={}, + encryption_context_from_request={}, + ) + mock_dec_materials.plaintext_data_key = plaintext_data_key + + mock_keyring.on_decrypt.return_value = mock_dec_materials + + # This should fail with NotImplementedError since V3 decryption isn't implemented yet + with pytest.raises(NotImplementedError, match="V3 decryption not yet implemented"): + pipeline.decrypt(mock_response, bucket="test-bucket", key="test-key") + + # Verify instruction file was fetched + mock_s3_client.get_object.assert_called_once_with( + Bucket="test-bucket", Key="test-key.instruction" + ) + + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=test + ##% The S3EC SHOULD support providing a custom Instruction File suffix + ##% on GetObject requests, regardless of whether or not re-encryption is supported. + def test_decrypt_with_custom_instruction_file_suffix(self): + """Test that a custom instruction file suffix is used when provided.""" + object_metadata = {} + + instruction_file_metadata = { + "x-amz-iv": base64.b64encode(os.urandom(12)).decode("utf-8"), + "x-amz-key-v2": base64.b64encode(b"encrypted-key-data").decode("utf-8"), + "x-amz-wrap-alg": "kms+context", + "x-amz-matdesc": json.dumps({"kms_cmk_id": "test-key-id"}), + "x-amz-cek-alg": "AES/GCM/NoPadding", + "x-amz-tag-len": "128", + "x-amz-crypto-instr-file": "", + } + + mock_s3_client = Mock() + mock_s3_client.get_object.return_value = { + "Body": BytesIO(b""), + "Metadata": instruction_file_metadata, + } + + mock_keyring = Mock(spec=S3Keyring) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + pipeline = GetEncryptedObjectPipeline(cmm, mock_s3_client) + + mock_response = { + "Body": BytesIO(b"encrypted-test-data"), + "Metadata": object_metadata, + } + + mock_keyring.on_decrypt.side_effect = Exception( + "Keyring called - instruction file was fetched" + ) + + with pytest.raises(Exception, match="Keyring called"): + pipeline.decrypt( + mock_response, + bucket="test-bucket", + key="test-key", + instruction_suffix=".custom-suffix", + ) + + mock_s3_client.get_object.assert_called_once_with( + Bucket="test-bucket", Key="test-key.custom-suffix" + ) diff --git a/test/test_s3_encryption_client_plugin.py b/test/test_s3_encryption_client_plugin.py new file mode 100644 index 00000000..bdc48c79 --- /dev/null +++ b/test/test_s3_encryption_client_plugin.py @@ -0,0 +1,141 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for S3EncryptionClientPlugin event handlers.""" + +import io +import json +from unittest.mock import Mock + +import pytest +from botocore.response import StreamingBody + +from s3_encryption import S3EncryptionClientConfig, S3EncryptionClientPlugin +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.materials.keyring import S3Keyring + + +class TestS3EncryptionClientPlugin: + """S3EncryptionClientPlugin event handler behavior.""" + + def test_instruction_file_mode_parses_instruction_file(self): + """Test that plaintext mode parses instruction file and returns metadata.""" + # Create plugin + mock_keyring = Mock(spec=S3Keyring) + config = S3EncryptionClientConfig(keyring=mock_keyring) + plugin = S3EncryptionClientPlugin(config) + + # Set plaintext mode + plugin._context.instruction_file_mode = True + plugin._context.key = "test-key.instruction" + + # Create instruction file body + instruction_metadata = { + "x-amz-iv": "test-iv", + "x-amz-key-v2": "test-key", + "x-amz-wrap-alg": "kms+context", + "x-amz-cek-alg": "AES/GCM/NoPadding", + } + instruction_body = json.dumps(instruction_metadata).encode("utf-8") + + # Create parsed response with instruction file marker in S3 metadata + parsed = { + "Body": StreamingBody(io.BytesIO(instruction_body), len(instruction_body)), + "Metadata": {"x-amz-crypto-instr-file": ""}, + } + + # Call event handler + plugin.on_get_object_after_call(parsed) + + # Verify metadata was updated with parsed instruction file + assert parsed["Metadata"]["x-amz-iv"] == "test-iv" + assert parsed["Metadata"]["x-amz-key-v2"] == "test-key" + assert parsed["Metadata"]["x-amz-wrap-alg"] == "kms+context" + assert parsed["Metadata"]["x-amz-cek-alg"] == "AES/GCM/NoPadding" + assert parsed["Metadata"]["x-amz-crypto-instr-file"] == "" + + # Verify body was cleared + assert parsed["Body"].read() == b"" + + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=test + ##% The content metadata stored in the Instruction File MUST be serialized to a JSON string. + def test_instruction_file_mode_invalid_json_raises_error(self): + """Test that invalid JSON in instruction file raises error.""" + # Create plugin + mock_keyring = Mock(spec=S3Keyring) + config = S3EncryptionClientConfig(keyring=mock_keyring) + plugin = S3EncryptionClientPlugin(config) + + # Set plaintext mode + plugin._context.instruction_file_mode = True + plugin._context.key = "test-key.instruction" + + # Create invalid JSON body + invalid_body = b"not valid json" + + # Create parsed response + parsed = { + "Body": StreamingBody(io.BytesIO(invalid_body), len(invalid_body)), + "Metadata": {"x-amz-crypto-instr-file": ""}, + } + + # Should raise error + with pytest.raises(S3EncryptionClientError, match="Instruction file is not valid JSON"): + plugin.on_get_object_after_call(parsed) + + def test_instruction_file_mode_non_dict_json_raises_error(self): + """Test that non-dict JSON in instruction file raises error.""" + # Create plugin + mock_keyring = Mock(spec=S3Keyring) + config = S3EncryptionClientConfig(keyring=mock_keyring) + plugin = S3EncryptionClientPlugin(config) + + # Set plaintext mode + plugin._context.instruction_file_mode = True + plugin._context.key = "test-key.instruction" + + # Create JSON array instead of object + invalid_body = json.dumps(["not", "a", "dict"]).encode("utf-8") + + # Create parsed response + parsed = { + "Body": StreamingBody(io.BytesIO(invalid_body), len(invalid_body)), + "Metadata": {"x-amz-crypto-instr-file": ""}, + } + + # Should raise error + with pytest.raises( + S3EncryptionClientError, match="Instruction file must contain a JSON object" + ): + plugin.on_get_object_after_call(parsed) + + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=test + ##% The serialized JSON string MUST be the only contents of the Instruction File. + def test_instruction_file_mode_invalid_keys_raises_error(self): + """Test that invalid keys in instruction file raises error.""" + # Create plugin + mock_keyring = Mock(spec=S3Keyring) + config = S3EncryptionClientConfig(keyring=mock_keyring) + plugin = S3EncryptionClientPlugin(config) + + # Set plaintext mode + plugin._context.instruction_file_mode = True + plugin._context.key = "test-key.instruction" + + # Create instruction file with invalid keys + instruction_metadata = { + "x-amz-iv": "test-iv", + "invalid-key": "should-not-be-here", + } + instruction_body = json.dumps(instruction_metadata).encode("utf-8") + + # Create parsed response + parsed = { + "Body": StreamingBody(io.BytesIO(instruction_body), len(instruction_body)), + "Metadata": {"x-amz-crypto-instr-file": ""}, + } + + # Should raise error + with pytest.raises(S3EncryptionClientError, match="Instruction file contains invalid keys"): + plugin.on_get_object_after_call(parsed) From 268bd4b219e34daa40f865435b35f8eec1d88a1a Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Wed, 18 Mar 2026 14:17:22 -0700 Subject: [PATCH 59/81] feat: add Key Committing AES-GCM and AES-CBC support (#147) Implement encryption, decryption, key commitment, CBC legacy support; add test server test for KC encrypt path; consolidate crypto params into AlgorithmSuite; tighten pipeline APIs and validation; add spec annotations, unit tests, and integration tests. --- .../decryption_exceptions.md | 49 ++ .../encryption_exceptions.md | 63 +++ src/s3_encryption/__init__.py | 106 +++- src/s3_encryption/key_derivation.py | 163 ++++++ src/s3_encryption/materials/__init__.py | 4 +- src/s3_encryption/materials/kms_keyring.py | 16 +- src/s3_encryption/materials/materials.py | 171 ++++++ src/s3_encryption/metadata.py | 16 +- src/s3_encryption/pipelines.py | 431 +++++++++++++-- ...eyCommitmentPolicyEncryptFailureTests.java | 91 ++++ .../amazon/encryption/s3/TestUtils.java | 50 +- test-server/python-v3-server/src/main.py | 36 +- test/integration/test_i_s3_encryption.py | 510 +++++------------- .../test_i_s3_encryption_instruction_file.py | 40 +- .../test_i_s3_encryption_multithreaded.py | 19 +- test/test_decryption.py | 390 ++++++++++++++ test/test_default_algorithm_commitment.py | 95 ++++ test/test_encryption.py | 240 +++++++++ test/test_encryption_materials_integration.py | 2 +- test/test_key_commitment.py | 157 ++++++ test/test_key_commitment_encrypt.py | 108 ++++ test/test_key_derivation.py | 281 ++++++++++ test/test_kms_keyring.py | 2 +- test/test_metadata.py | 5 +- test/test_pipelines.py | 76 ++- 25 files changed, 2662 insertions(+), 459 deletions(-) create mode 100644 compliance_exceptions/decryption_exceptions.md create mode 100644 compliance_exceptions/encryption_exceptions.md create mode 100644 src/s3_encryption/key_derivation.py create mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java create mode 100644 test/test_decryption.py create mode 100644 test/test_default_algorithm_commitment.py create mode 100644 test/test_encryption.py create mode 100644 test/test_key_commitment.py create mode 100644 test/test_key_commitment_encrypt.py create mode 100644 test/test_key_derivation.py diff --git a/compliance_exceptions/decryption_exceptions.md b/compliance_exceptions/decryption_exceptions.md new file mode 100644 index 00000000..4919f2ce --- /dev/null +++ b/compliance_exceptions/decryption_exceptions.md @@ -0,0 +1,49 @@ +# Compliance Exceptions for Decryption Implementation + +## Summary + +The Python S3 Encryption Client does not currently support Ranged Gets. +Ranged Gets allow downloading and decrypting a subset of bytes from an encrypted S3 object. +This is an optional feature per the specification ("MAY support") and is planned for a future release. + +## Ranged Gets + +##= specification/s3-encryption/decryption.md#ranged-gets +##= type=exception +##% The S3EC MAY support the "range" parameter on GetObject which specifies a subset of bytes to download and decrypt. + +Justification: Ranged Gets are not yet implemented in the Python S3 Encryption Client. The specification uses MAY, making this an optional feature. This is planned for a future release. + +--- + +##= specification/s3-encryption/decryption.md#ranged-gets +##= type=exception +##% If the S3EC supports Ranged Gets, the S3EC MUST adjust the customer-provided range to include the beginning and end of the cipher blocks for the given range. + +Justification: Not applicable since Ranged Gets are not yet supported. When Ranged Gets are implemented, this requirement will be fulfilled. + +--- + +##= specification/s3-encryption/decryption.md#ranged-gets +##= type=exception +##% If the object was encrypted with ALG_AES_256_GCM_IV12_TAG16_NO_KDF, then ALG_AES_256_CTR_IV16_TAG16_NO_KDF MUST be used to decrypt the range of the object. + +Justification: Not applicable since Ranged Gets are not yet supported. When Ranged Gets are implemented, the correct CTR-mode algorithm suite will be used for GCM-encrypted objects. + +--- + +##= specification/s3-encryption/decryption.md#ranged-gets +##= type=exception +##% If the object was encrypted with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, then ALG_AES_256_CTR_HKDF_SHA512_COMMIT_KEY MUST be used to decrypt the range of the object. + +Justification: Not applicable since Ranged Gets are not yet supported. When Ranged Gets are implemented, the correct CTR-mode algorithm suite will be used for key-committing objects. + +--- + +##= specification/s3-encryption/decryption.md#ranged-gets +##= type=exception +##% If the GetObject response contains a range, but the GetObject request does not contain a range, the S3EC MUST throw an exception. + +Justification: Not applicable since Ranged Gets are not yet supported. When Ranged Gets are implemented, this validation will be added to detect unexpected range responses. + +--- diff --git a/compliance_exceptions/encryption_exceptions.md b/compliance_exceptions/encryption_exceptions.md new file mode 100644 index 00000000..bf7d9f62 --- /dev/null +++ b/compliance_exceptions/encryption_exceptions.md @@ -0,0 +1,63 @@ +# Compliance Exceptions for Encryption Implementation + +## Summary + +The Python S3 Encryption Client does not implement AES-CTR algorithm suites (used only for ranged-get decryption), +does not yet validate IV/Message ID for zero values, does not validate maximum plaintext length, +and relies on Python's `cryptography` library to automatically append GCM auth tags. + +## AES-CTR Algorithm Suites + +##= specification/s3-encryption/encryption.md#alg-aes-256-ctr-hkdf-sha512-commit-key +##= type=exception +##% Attempts to encrypt using key committing AES-CTR MUST fail. + +Justification: The AES-CTR algorithm suites are only used for ranged-get decryption. Since ranged gets are not yet implemented, these algorithm suites are not defined in the `AlgorithmSuite` enum and cannot be selected for encryption. The constraint is satisfied structurally. + +--- + +##= specification/s3-encryption/encryption.md#alg-aes-256-ctr-iv16-tag16-no-kdf +##= type=exception +##% Attempts to encrypt using AES-CTR MUST fail. + +Justification: Same as above. AES-CTR is not available as an algorithm suite option, so it cannot be used for encryption. + +--- + +## GCM Auth Tag Appending + +##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key +##= type=exception +##% The client MUST append the GCM auth tag to the ciphertext if the underlying crypto provider does not do so automatically. + +Justification: Python's `cryptography` library (`AESGCM.encrypt`) automatically appends the GCM authentication tag to the ciphertext. No manual appending is needed. + +--- + +##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf +##= type=exception +##% The client MUST append the GCM auth tag to the ciphertext if the underlying crypto provider does not do so automatically. + +Justification: Python's `cryptography` library (`AESGCM.encrypt`) automatically appends the GCM authentication tag to the ciphertext. No manual appending is needed. + +--- + +## Cipher Initialization Validation + +##= specification/s3-encryption/encryption.md#cipher-initialization +##= type=exception +##% The client SHOULD validate that the generated IV or Message ID is not zeros. + +Justification: This SHOULD-level validation is not yet implemented. The IV and Message ID are generated using `os.urandom()`, which is cryptographically secure and extremely unlikely to produce all-zero output. This validation is planned for a future release. + +--- + +## Plaintext Length Validation + +##= specification/s3-encryption/encryption.md#content-encryption +##= type=exception +##% The client MUST validate that the length of the plaintext bytes does not exceed the algorithm suite's cipher's maximum content length in bytes. + +Justification: Maximum plaintext length validation is not yet implemented. For AES-GCM with a 12-byte IV, the maximum plaintext size is approximately 64 GiB, which exceeds practical S3 single-object upload limits. This validation is planned for a future release. + +--- diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index a3558195..f2af6d7a 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -15,16 +15,39 @@ DefaultCryptoMaterialsManager, ) from .materials.keyring import AbstractKeyring +from .materials.materials import AlgorithmSuite, CommitmentPolicy from .pipelines import GetEncryptedObjectPipeline, PutEncryptedObjectPipeline S3_METADATA_PREFIX = "x-amz-meta-" +# Thread-local context attribute names +_CTX_ENCRYPTION_CONTEXT = "encryption_context" +_CTX_BUCKET = "bucket" +_CTX_KEY = "key" +_CTX_S3_CLIENT = "s3_client" +_CTX_INSTRUCTION_FILE_MODE = "instruction_file_mode" + +# Attributes to clean up after get_object completes +# (s3_client is intentionally excluded — it is not request-scoped) +_GET_OBJECT_CLEANUP_ATTRS = (_CTX_ENCRYPTION_CONTEXT, _CTX_BUCKET, _CTX_KEY) + @define class S3EncryptionClientConfig: """Configuration object for the S3 Encryption Client.""" keyring: AbstractKeyring + encryption_algorithm: AlgorithmSuite = field( + default=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) + commitment_policy: CommitmentPolicy = field( + default=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + ) + ##= specification/s3-encryption/client.md#enable-legacy-unauthenticated-modes + ##% The S3EC MUST support the option to enable or disable legacy unauthenticated modes (content encryption algorithms). + ##= specification/s3-encryption/client.md#enable-legacy-unauthenticated-modes + ##% The option to enable legacy unauthenticated modes MUST be set to false by default. + enable_legacy_unauthenticated_modes: bool = field(default=False) cmm: AbstractCryptoMaterialsManager = field() ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file ##= type=implementation @@ -41,6 +64,51 @@ class S3EncryptionClientConfig: def _default_cmm_for_keyring(self): return DefaultCryptoMaterialsManager(self.keyring) + ##= specification/s3-encryption/client.md#encryption-algorithm + ##% The S3EC MUST validate that the configured encryption algorithm is not legacy. + ##= specification/s3-encryption/client.md#encryption-algorithm + ##% If the configured encryption algorithm is legacy, then the S3EC MUST throw an exception. + ##= specification/s3-encryption/client.md#key-commitment + ##% The S3EC MUST validate the configured Encryption Algorithm against the provided key commitment policy. + ##= specification/s3-encryption/client.md#key-commitment + ##% If the configured Encryption Algorithm is incompatible with the key commitment policy, then it MUST throw an exception. + def __attrs_post_init__(self): + """Validate algorithm suite and commitment policy configuration.""" + if self.encryption_algorithm.is_legacy: + raise S3EncryptionClientError( + f"Cannot configure S3 Encryption Client with legacy algorithm suite " + f"{self.encryption_algorithm.name}. Legacy algorithm suites are only " + f"supported for decryption (and enable_legacy_unauthenticated_modes is True)." + ) + + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##% When the commitment policy is REQUIRE_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST only encrypt using an algorithm suite which supports key commitment. + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##% When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, the S3EC MUST only encrypt using an algorithm suite which supports key commitment. + if ( + self.commitment_policy + in ( + CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + and not self.encryption_algorithm.supports_key_commitment + ): + raise S3EncryptionClientError( + f"Commitment policy {self.commitment_policy.name} requires a key-committing " + f"algorithm suite, but {self.encryption_algorithm.name} does not support key commitment." + ) + + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##% When the commitment policy is FORBID_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST NOT encrypt using an algorithm suite which supports key commitment. + if ( + self.commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT + and self.encryption_algorithm.supports_key_commitment + ): + raise S3EncryptionClientError( + f"Commitment policy {self.commitment_policy.name} forbids key-committing " + f"algorithm suites, but {self.encryption_algorithm.name} supports key commitment." + ) + class S3EncryptionClientPlugin: """Plugin that adds encryption/decryption capabilities to a boto3 S3 client. @@ -67,7 +135,7 @@ def on_put_object_before_call(self, params, **kwargs): params: Dictionary of parameters for the PutObject call (after serialization) **kwargs: Additional event arguments """ - if getattr(self._context, "instruction_file_mode", False): + if getattr(self._context, _CTX_INSTRUCTION_FILE_MODE, False): raise S3EncryptionClientError( "Instruction file mode is exclusively for reading instruction files " "and not supported in put_object!" @@ -88,11 +156,12 @@ def on_put_object_before_call(self, params, **kwargs): # Unexpected body type - should not happen as boto3 validates before this point raise S3EncryptionClientError("Unexpected type of body parameter!") - encryption_context = getattr(self._context, "encryption_context", None) + encryption_context = getattr(self._context, _CTX_ENCRYPTION_CONTEXT, None) - pipeline = PutEncryptedObjectPipeline(self.config.cmm) + pipeline = PutEncryptedObjectPipeline(self.config.cmm, self.config.encryption_algorithm) encrypted_data, encryption_metadata = pipeline.encrypt( - body_bytes, encryption_context=encryption_context + body_bytes, + encryption_context=encryption_context, ) params["body"] = encrypted_data @@ -118,12 +187,12 @@ def on_get_object_after_call(self, parsed, **kwargs): **kwargs: Additional event arguments (includes 'params' with request parameters) """ # Check if plaintext mode is enabled via thread-local flag - if getattr(self._context, "instruction_file_mode", False): + if getattr(self._context, _CTX_INSTRUCTION_FILE_MODE, False): self.process_instruction_file(parsed) return # Get encryption context from thread-local storage (set by get_object wrapper) - encryption_context = getattr(self._context, "encryption_context", None) + encryption_context = getattr(self._context, _CTX_ENCRYPTION_CONTEXT, None) # The parsed response already has the Body as a StreamingBody # We need to read it, decrypt it, and replace it @@ -137,13 +206,15 @@ def on_get_object_after_call(self, parsed, **kwargs): # Create a pipeline and decrypt the data pipeline = GetEncryptedObjectPipeline( self.config.cmm, - s3_client=getattr(self._context, "s3_client", None), + commitment_policy=self.config.commitment_policy, + s3_client=getattr(self._context, _CTX_S3_CLIENT, None), + enable_legacy_unauthenticated_modes=self.config.enable_legacy_unauthenticated_modes, ) decrypted_data = pipeline.decrypt( response, encryption_context, - bucket=getattr(self._context, "bucket", None), - key=getattr(self._context, "key", None), + bucket=getattr(self._context, _CTX_BUCKET, None), + key=getattr(self._context, _CTX_KEY, None), instruction_suffix=self.config.instruction_file_suffix, ) @@ -163,7 +234,7 @@ def process_instruction_file(self, parsed): Args: parsed: Dictionary containing the parsed response """ - instruction_key = getattr(self._context, "key", None) + instruction_key = getattr(self._context, _CTX_KEY, None) # In plaintext mode, parse instruction file and append to metadata existing_metadata = parsed.get("Metadata", {}) @@ -242,8 +313,8 @@ def put_object(self, **kwargs): raise S3EncryptionClientError(f"Failed to encrypt object: {str(e)}") from e finally: # Clean up thread-local storage - if hasattr(self._plugin._context, "encryption_context"): - delattr(self._plugin._context, "encryption_context") + if hasattr(self._plugin._context, _CTX_ENCRYPTION_CONTEXT): + delattr(self._plugin._context, _CTX_ENCRYPTION_CONTEXT) def get_object(self, **kwargs): """Download and decrypt an object from S3. @@ -266,12 +337,12 @@ def get_object(self, **kwargs): encryption_context = kwargs.pop("EncryptionContext", None) # Store encryption context in thread-local storage for the event handler - self._plugin._context.encryption_context = encryption_context + setattr(self._plugin._context, _CTX_ENCRYPTION_CONTEXT, encryption_context) # Store wrapped client in thread-local storage for # the event handler to fetch instruction files - self._plugin._context.s3_client = self.wrapped_s3_client - self._plugin._context.bucket = kwargs.get("Bucket") - self._plugin._context.key = kwargs.get("Key") + setattr(self._plugin._context, _CTX_S3_CLIENT, self.wrapped_s3_client) + setattr(self._plugin._context, _CTX_BUCKET, kwargs.get("Bucket")) + setattr(self._plugin._context, _CTX_KEY, kwargs.get("Key")) try: return self.wrapped_s3_client.get_object(**kwargs) @@ -284,7 +355,6 @@ def get_object(self, **kwargs): finally: # Clean up thread-local storage; # do not clean up the client as it is not thread local only - attrs = ["encryption_context", "Bucket", "Key"] - for attr in attrs: + for attr in _GET_OBJECT_CLEANUP_ATTRS: if hasattr(self._plugin._context, attr): delattr(self._plugin._context, attr) diff --git a/src/s3_encryption/key_derivation.py b/src/s3_encryption/key_derivation.py new file mode 100644 index 00000000..8183f5f3 --- /dev/null +++ b/src/s3_encryption/key_derivation.py @@ -0,0 +1,163 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Key derivation for S3 Encryption Client key-committing algorithm suites. + +Implements HKDF-based key derivation as specified in: + specification/s3-encryption/key-derivation.md + +For ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + - Extract: HKDF-SHA512, salt = Message ID (28 bytes), IKM = plaintext data key + - Expand (DEK): info = suite_id_bytes + b"DERIVEKEY", output = 32 bytes + - Expand (Commit Key): info = suite_id_bytes + b"COMMITKEY", output = 28 bytes +""" + +from __future__ import annotations + +import hmac +from typing import TYPE_CHECKING + +from cryptography.hazmat.primitives.hashes import SHA512 +from cryptography.hazmat.primitives.kdf.hkdf import HKDFExpand + +from .exceptions import S3EncryptionClientError, S3EncryptionClientSecurityError + +if TYPE_CHECKING: + from .materials.materials import AlgorithmSuite + +# Map of supported KDF hash algorithm names to cryptography hash classes. +_HASH_ALGORITHMS = { + "sha512": SHA512, +} + + +def _hkdf_extract(salt: bytes, ikm: bytes, hash_algorithm: str) -> bytes: + """HKDF extract step using HMAC. + + Args: + salt: The salt value (Message ID). + ikm: Input keying material (plaintext data key). + hash_algorithm: Hash algorithm name (e.g. "sha512"). + + Returns: + The pseudorandom key (PRK). + """ + return hmac.new(salt, ikm, hash_algorithm).digest() + + +def _hkdf_expand(prk: bytes, info: bytes, length: int, hash_algorithm: str) -> bytes: + """HKDF expand step. + + Args: + prk: Pseudorandom key from extract step. + info: Context/application-specific info string. + length: Desired output length in bytes. + hash_algorithm: Hash algorithm name (e.g. "sha512"). + + Returns: + Output keying material of the requested length. + + Raises: + S3EncryptionClientError: If the hash algorithm is not supported. + """ + hash_cls = _HASH_ALGORITHMS.get(hash_algorithm) + if hash_cls is None: + raise S3EncryptionClientError(f"Unsupported KDF hash algorithm: {hash_algorithm}") + hkdf = HKDFExpand(algorithm=hash_cls(), length=length, info=info) + return hkdf.derive(prk) + + +def derive_keys( + plaintext_data_key: bytes, + message_id: bytes, + algorithm_suite: AlgorithmSuite, +) -> tuple[bytes, bytes]: + """Derive the encryption key and commitment key from a plaintext data key. + + Uses HKDF with SHA-512 as specified in the S3EC key derivation spec. + + Args: + plaintext_data_key: The plaintext data key from the keyring. + message_id: The generated Message ID used as the HKDF salt. + algorithm_suite: The algorithm suite whose parameters drive key lengths + and info strings. + + Returns: + A tuple of (derived_encryption_key, commit_key). + """ + suite_id = algorithm_suite.suite_id_bytes + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% - The length of the output keying material MUST equal the encryption key length specified by the algorithm suite encryption settings. + enc_key_len = algorithm_suite.data_key_length_bytes + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% - The length of the output keying material MUST equal the commit key length specified by the supported algorithm suites. + commit_key_len = algorithm_suite.commitment_length_bytes + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% - The hash function MUST be specified by the algorithm suite commitment settings. + hash_alg = algorithm_suite.kdf_hash_algorithm + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% - The length of the input keying material MUST equal the key derivation input length specified by the algorithm suite commit key derivation setting. + if len(plaintext_data_key) != enc_key_len: + raise S3EncryptionClientError( + f"Plaintext data key length ({len(plaintext_data_key)}) does not match " + f"the key derivation input length ({enc_key_len}) specified by the algorithm suite." + ) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% - The input keying material MUST be the plaintext data key (PDK) generated by the key provider. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% - The salt MUST be the Message ID with the length defined in the algorithm suite. + prk = _hkdf_extract(salt=message_id, ikm=plaintext_data_key, hash_algorithm=hash_alg) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% - The DEK input pseudorandom key MUST be the output from the extract step. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% - The input info MUST be a concatenation of the algorithm suite ID as bytes + ##% followed by the string DERIVEKEY as UTF8 encoded bytes. + derived_encryption_key = _hkdf_expand( + prk, info=suite_id + b"DERIVEKEY", length=enc_key_len, hash_algorithm=hash_alg + ) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% - The CK input pseudorandom key MUST be the output from the extract step. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% - The input info MUST be a concatenation of the algorithm suite ID as bytes + ##% followed by the string COMMITKEY as UTF8 encoded bytes. + commit_key = _hkdf_expand( + prk, info=suite_id + b"COMMITKEY", length=commit_key_len, hash_algorithm=hash_alg + ) + + return derived_encryption_key, commit_key + + +def verify_commitment(stored_commitment: bytes, derived_commitment: bytes) -> None: + """Verify key commitment in constant time. + + Args: + stored_commitment: The commitment value from the object metadata. + derived_commitment: The commitment value derived from the data key. + + Raises: + S3EncryptionClientSecurityError: If the commitment values do not match. + """ + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##= type=implementation + ##% When using an algorithm suite which supports key commitment, the verification of the derived key commitment value MUST be done in constant time. + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##= type=implementation + ##% When using an algorithm suite which supports key commitment, the client MUST throw an exception when the derived key commitment value + ##% and stored key commitment value do not match. + if not hmac.compare_digest(stored_commitment, derived_commitment): + raise S3EncryptionClientSecurityError( + "Key commitment verification failed: stored commitment does not match derived commitment." + ) diff --git a/src/s3_encryption/materials/__init__.py b/src/s3_encryption/materials/__init__.py index c67d5802..c5cc7d6d 100644 --- a/src/s3_encryption/materials/__init__.py +++ b/src/s3_encryption/materials/__init__.py @@ -10,7 +10,7 @@ from .encrypted_data_key import EncryptedDataKey from .keyring import AbstractKeyring from .kms_keyring import KmsKeyring -from .materials import EncryptionMaterials +from .materials import AlgorithmSuite, CommitmentPolicy, EncryptionMaterials __all__ = [ "AbstractKeyring", @@ -18,5 +18,7 @@ "AbstractCryptoMaterialsManager", "DefaultCryptoMaterialsManager", "EncryptedDataKey", + "AlgorithmSuite", + "CommitmentPolicy", "EncryptionMaterials", ] diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index a32493fc..edf1d27b 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -10,6 +10,7 @@ from botocore import client from ..exceptions import S3EncryptionClientError +from ..materials.materials import AlgorithmSuite from .encrypted_data_key import EncryptedDataKey from .keyring import S3Keyring @@ -76,7 +77,20 @@ def on_encrypt(self, enc_materials): ##= specification/s3-encryption/materials/s3-kms-keyring.md#supported-wrapping-algorithm-modes ##= type=implication ##% The KmsKeyring MUST NOT support encryption using KmsV1 mode. - encryption_context["aws:x-amz-cek-alg"] = "AES/GCM/NoPadding" + # For committing algorithm suites (V3), the encryption context algorithm + # value is the algorithm suite ID as a string ("115"), not the cipher name. + # For non-committing suites (V2), use the cipher name ("AES/GCM/NoPadding"). + if ( + enc_materials.encryption_algorithm + == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ): + encryption_context["aws:x-amz-cek-alg"] = str( + enc_materials.encryption_algorithm.suite_id + ) + else: + encryption_context["aws:x-amz-cek-alg"] = ( + enc_materials.encryption_algorithm.cipher_name + ) # Python implementation uses KMS GenerateDataKey instead of the spec's # EncryptDataKey pattern diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index 3966e17c..f2e8fd4f 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -7,6 +7,7 @@ and decryption operations. """ +from enum import Enum from typing import Any from attrs import define, field @@ -14,6 +15,172 @@ from .encrypted_data_key import EncryptedDataKey +class AlgorithmSuite(Enum): + """Algorithm suites supported by the S3 Encryption Client. + + Each member consolidates all cryptographic parameters for a given suite, + modeled after the Java reference implementation. The tuple values are: + + (id, is_legacy, data_key_algorithm, data_key_length_bits, + cipher_name, cipher_block_size_bits, cipher_iv_length_bits, + cipher_tag_length_bits, is_committing, commitment_length_bits, + commitment_nonce_length_bits, kdf_hash_algorithm, suite_id_bytes) + """ + + ALG_AES_256_CBC_IV16_NO_KDF = ( + 0x0070, # id + True, # is_legacy + "AES", # data_key_algorithm + 256, # data_key_length_bits + "AES/CBC/PKCS5Padding", # cipher_name + 128, # cipher_block_size_bits + 128, # cipher_iv_length_bits (16 bytes) + 0, # cipher_tag_length_bits (CBC has no auth tag) + False, # is_committing + 0, # commitment_length_bits + 0, # commitment_nonce_length_bits + None, # kdf_hash_algorithm + b"", # suite_id_bytes + ) + + ALG_AES_256_GCM_IV12_TAG16_NO_KDF = ( + 0x0072, # id + False, # is_legacy + "AES", # data_key_algorithm + 256, # data_key_length_bits + "AES/GCM/NoPadding", # cipher_name + 128, # cipher_block_size_bits + 96, # cipher_iv_length_bits (12 bytes) + 128, # cipher_tag_length_bits (16 bytes) + False, # is_committing + 0, # commitment_length_bits + 0, # commitment_nonce_length_bits + None, # kdf_hash_algorithm + b"", # suite_id_bytes + ) + + ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY = ( + 0x0073, # id + False, # is_legacy + "AES", # data_key_algorithm + 256, # data_key_length_bits + "AES/GCM/HKDF/CommitKey", # cipher_name + 128, # cipher_block_size_bits + 96, # cipher_iv_length_bits (12 bytes) + 128, # cipher_tag_length_bits (16 bytes) + True, # is_committing + 224, # commitment_length_bits (28 bytes) + 224, # commitment_nonce_length_bits (28 bytes = message_id) + "sha512", # kdf_hash_algorithm + b"\x00\x73", # suite_id_bytes + ) + + def __init__( + self, + suite_id: int, + is_legacy: bool, + data_key_algorithm: str, + data_key_length_bits: int, + cipher_name: str, + cipher_block_size_bits: int, + cipher_iv_length_bits: int, + cipher_tag_length_bits: int, + is_committing: bool, + commitment_length_bits: int, + commitment_nonce_length_bits: int, + kdf_hash_algorithm: str | None, + suite_id_bytes: bytes, + ): + """Initialize algorithm suite parameters from the enum tuple.""" + self._id = suite_id + self._is_legacy = is_legacy + self._data_key_algorithm = data_key_algorithm + self._data_key_length_bits = data_key_length_bits + self._cipher_name = cipher_name + self._cipher_block_size_bits = cipher_block_size_bits + self._cipher_iv_length_bits = cipher_iv_length_bits + self._cipher_tag_length_bits = cipher_tag_length_bits + self._is_committing = is_committing + self._commitment_length_bits = commitment_length_bits + self._commitment_nonce_length_bits = commitment_nonce_length_bits + self._kdf_hash_algorithm = kdf_hash_algorithm + self._suite_id_bytes = suite_id_bytes + + # --- Convenience properties --- + + @property + def suite_id(self) -> int: + """Numeric identifier for this algorithm suite.""" + return self._id + + @property + def is_legacy(self) -> bool: + """Return True if this algorithm suite is a legacy unauthenticated mode.""" + return self._is_legacy + + @property + def supports_key_commitment(self) -> bool: + """Return True if this algorithm suite supports key commitment.""" + return self._is_committing + + @property + def data_key_length_bytes(self) -> int: + """Data key length in bytes.""" + return self._data_key_length_bits // 8 + + @property + def cipher_name(self) -> str: + """Cipher transformation string (e.g. 'AES/GCM/NoPadding').""" + return self._cipher_name + + @property + def cipher_iv_length_bytes(self) -> int: + """Initialization vector length in bytes.""" + return self._cipher_iv_length_bits // 8 + + @property + def commitment_length_bytes(self) -> int: + """Key commitment value length in bytes.""" + return self._commitment_length_bits // 8 + + @property + def commitment_nonce_length_bytes(self) -> int: + """Length of the message ID / HKDF salt in bytes.""" + return self._commitment_nonce_length_bits // 8 + + @property + def suite_id_bytes(self) -> bytes: + """Algorithm suite ID as raw bytes for use in HKDF info strings.""" + return self._suite_id_bytes + + @property + def kdf_hash_algorithm(self) -> str | None: + """Hash algorithm name for HKDF, usable with hmac (e.g. 'sha512').""" + return self._kdf_hash_algorithm + + @property + def kc_gcm_iv(self) -> bytes: + """Fixed IV for key-committing GCM: all 0x01 bytes of cipher_iv_length.""" + if not self._is_committing: + raise ValueError(f"{self.name} does not support key commitment") + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% The IV's total length MUST match the IV length defined by the algorithm suite. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% When encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ##% the IV used in the AES-GCM content encryption/decryption MUST consist entirely of bytes with the value 0x01. + return b"\x01" * self.cipher_iv_length_bytes + + +class CommitmentPolicy(Enum): + """Commitment policies controlling key-commitment behavior.""" + + FORBID_ENCRYPT_ALLOW_DECRYPT = "ForbidEncryptAllowDecrypt" + REQUIRE_ENCRYPT_ALLOW_DECRYPT = "RequireEncryptAllowDecrypt" + REQUIRE_ENCRYPT_REQUIRE_DECRYPT = "RequireEncryptRequireDecrypt" + + @define class EncryptionMaterials: """Class representing encryption materials for S3 encryption. @@ -27,6 +194,9 @@ class EncryptionMaterials: plaintext_data_key (Optional[bytes]): The plaintext data key """ + encryption_algorithm: AlgorithmSuite = field( + default=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) encryption_context: dict[str, str] = field(factory=dict) encrypted_data_key: EncryptedDataKey | None = field(default=None) plaintext_data_key: bytes | None = field(default=None) @@ -87,6 +257,7 @@ class DecryptionMaterials: encryption_context_stored: dict[str, str] = field(factory=dict) encryption_context_from_request: dict[str, str] = field(factory=dict) plaintext_data_key: bytes | None = field(default=None) + algorithm_suite: AlgorithmSuite | None = field(default=None) @classmethod def from_dict(cls, materials_dict: dict[str, Any]) -> "DecryptionMaterials": diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py index 5c8bbda3..0b0fbce6 100644 --- a/src/s3_encryption/metadata.py +++ b/src/s3_encryption/metadata.py @@ -51,8 +51,8 @@ class ObjectMetadata: # V3 format fields (compressed) content_cipher_v3: str | None = field(default=None) encrypted_data_key_v3: str | None = field(default=None) - mat_desc_v3: str | None = field(default=None) - encryption_context_v3: str | None = field(default=None) + mat_desc_v3: str | dict | None = field(default=None) + encryption_context_v3: str | dict | None = field(default=None) encrypted_data_key_algorithm_v3: str | None = field(default=None) key_commitment_v3: str | None = field(default=None) message_id_v3: str | None = field(default=None) @@ -137,7 +137,7 @@ def to_dict(self) -> dict[str, str]: if self.content_cipher is not None: result[self.CONTENT_CIPHER] = self.content_cipher - if self.content_cipher_tag_length is not None: + if self.content_cipher_tag_length is not None and not self.is_v3_format(): result[self.CONTENT_CIPHER_TAG_LENGTH] = self.content_cipher_tag_length if self.instruction_file is not None: @@ -150,10 +150,16 @@ def to_dict(self) -> dict[str, str]: result[self.ENCRYPTED_DATA_KEY_V3] = self.encrypted_data_key_v3 if self.mat_desc_v3 is not None: - result[self.MAT_DESC_V3] = self.mat_desc_v3 + if isinstance(self.mat_desc_v3, dict): + result[self.MAT_DESC_V3] = json.dumps(self.mat_desc_v3) + else: + result[self.MAT_DESC_V3] = self.mat_desc_v3 if self.encryption_context_v3 is not None: - result[self.ENCRYPTION_CONTEXT_V3] = self.encryption_context_v3 + if isinstance(self.encryption_context_v3, dict): + result[self.ENCRYPTION_CONTEXT_V3] = json.dumps(self.encryption_context_v3) + else: + result[self.ENCRYPTION_CONTEXT_V3] = self.encryption_context_v3 if self.encrypted_data_key_algorithm_v3 is not None: result[self.ENCRYPTED_DATA_KEY_ALGORITHM_V3] = self.encrypted_data_key_algorithm_v3 diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 02a5a9c9..d0e9ba79 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -7,16 +7,25 @@ """ import base64 +import json import os from attrs import define, field +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.padding import PKCS7 -from .exceptions import S3EncryptionClientError +from .exceptions import S3EncryptionClientError, S3EncryptionClientSecurityError from .instruction_file import fetch_instruction_file +from .key_derivation import derive_keys, verify_commitment from .materials.crypto_materials_manager import AbstractCryptoMaterialsManager from .materials.encrypted_data_key import EncryptedDataKey -from .materials.materials import DecryptionMaterials, EncryptionMaterials +from .materials.materials import ( + AlgorithmSuite, + CommitmentPolicy, + DecryptionMaterials, + EncryptionMaterials, +) from .metadata import ObjectMetadata @@ -29,6 +38,7 @@ class PutEncryptedObjectPipeline: """ cmm: AbstractCryptoMaterialsManager = field() + encryption_algorithm: AlgorithmSuite = field() def encrypt(self, plaintext, encryption_context=None): """Encrypt the data before it is stored in S3. @@ -41,34 +51,53 @@ def encrypt(self, plaintext, encryption_context=None): bytes: The encrypted data dict: Metadata about the encryption to be stored with the object """ - # Create encryption materials request with encryption context copy + ##= specification/s3-encryption/encryption.md#content-encryption + ##= type=implementation + ##% The S3EC MUST use the encryption algorithm configured during + ##% [client](./client.md) initialization. enc_mats_request = EncryptionMaterials( - encryption_context={} if encryption_context is None else encryption_context.copy() + encryption_algorithm=self.encryption_algorithm, + encryption_context={} if encryption_context is None else encryption_context.copy(), ) # Get encryption materials from the crypto materials manager enc_mats = self.cmm.get_encryption_materials(enc_mats_request) - # Generate initialization vector - iv = os.urandom(12) - - # Encrypt the data if enc_mats.plaintext_data_key is None: raise RuntimeError("No plaintext data key found!") - - aesgcm = AESGCM(enc_mats.plaintext_data_key) - ciphertext = aesgcm.encrypt(nonce=iv, data=plaintext, associated_data=None) - encrypted_data = ciphertext - b64_iv = base64.b64encode(iv).decode("utf-8") - - # Get the encrypted data key if enc_mats.encrypted_data_key is None: raise RuntimeError("No encrypted data key found!") edk_bytes = enc_mats.encrypted_data_key.encrypted_data_key + + if self.encryption_algorithm == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + return self._encrypt_kc_gcm(plaintext, enc_mats, edk_bytes) + return self._encrypt_gcm(plaintext, enc_mats, edk_bytes) + + def _encrypt_gcm(self, plaintext, enc_mats, edk_bytes): + """Encrypt using ALG_AES_256_GCM_IV12_TAG16_NO_KDF (V2 format).""" + ##= specification/s3-encryption/encryption.md#content-encryption + ##= type=implementation + ##% The client MUST generate an IV or Message ID using the length of the IV + ##% or Message ID defined in the algorithm suite. + iv = os.urandom(enc_mats.encryption_algorithm.cipher_iv_length_bytes) + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf + ##= type=implementation + ##% The client MUST initialize the cipher, or call an AES-GCM encryption API, + ##% with the plaintext data key, the generated IV, and the tag length defined + ##% in the Algorithm Suite when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + aesgcm = AESGCM(enc_mats.plaintext_data_key) + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf + ##= type=implementation + ##% The client MUST NOT provide any AAD when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + encrypted_data = aesgcm.encrypt(nonce=iv, data=plaintext, associated_data=None) + + b64_iv = base64.b64encode(iv).decode("utf-8") b64_edk = base64.b64encode(edk_bytes).decode("utf-8") - # Create metadata using the ObjectMetadata class + ##= specification/s3-encryption/encryption.md#content-encryption + ##= type=implementation + ##% The generated IV or Message ID MUST be set or returned from the encryption metadata = ObjectMetadata( encrypted_data_key_v2=b64_edk, encrypted_data_key_algorithm="kms+context", @@ -77,10 +106,63 @@ def encrypt(self, plaintext, encryption_context=None): encrypted_data_key_context=enc_mats.encryption_context, ) - # Convert to dictionary for storage in S3 metadata - encryption_metadata = metadata.to_dict() + return encrypted_data, metadata.to_dict() + + def _encrypt_kc_gcm(self, plaintext, enc_mats, edk_bytes): + """Encrypt using ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY (V3 format).""" + ##= specification/s3-encryption/encryption.md#content-encryption + ##= type=implementation + ##% The client MUST generate an IV or Message ID using the length of the IV + ##% or Message ID defined in the algorithm suite. + algorithm_suite = enc_mats.encryption_algorithm + message_id = os.urandom(algorithm_suite.commitment_nonce_length_bytes) + + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key + ##= type=implementation + ##% The client MUST use HKDF to derive the key commitment value and the derived + ##% encrypting key as described in [Key Derivation](key-derivation.md). + derived_encryption_key, commit_key = derive_keys( + enc_mats.plaintext_data_key, message_id, algorithm_suite + ) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% The client MUST initialize the cipher, or call an AES-GCM encryption API, with the derived encryption key, an IV containing only bytes with the value 0x01, + ##% and the tag length defined in the Algorithm Suite when encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. + aesgcm = AESGCM(derived_encryption_key) + encrypted_data = aesgcm.encrypt( + nonce=algorithm_suite.kc_gcm_iv, + data=plaintext, + associated_data=algorithm_suite.suite_id_bytes, + ) + + b64_edk = base64.b64encode(edk_bytes).decode("utf-8") + b64_message_id = base64.b64encode(message_id).decode("utf-8") + b64_commit_key = base64.b64encode(commit_key).decode("utf-8") + + ##= specification/s3-encryption/encryption.md#content-encryption + ##= type=implementation + ##% The generated IV or Message ID MUST be set or returned from the encryption + ##% process such that it can be included in the content metadata. + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key + ##= type=implementation + ##% The derived key commitment value MUST be set or returned from the encryption + ##% process such that it can be included in the content metadata. + metadata = ObjectMetadata( + content_cipher_v3=str(algorithm_suite.suite_id), + encrypted_data_key_algorithm_v3="12", + encrypted_data_key_v3=b64_edk, + message_id_v3=b64_message_id, + key_commitment_v3=b64_commit_key, + encryption_context_v3=( + enc_mats.encryption_context if enc_mats.encryption_context else None + ), + ) - return encrypted_data, encryption_metadata + return encrypted_data, metadata.to_dict() @define @@ -92,7 +174,50 @@ class GetEncryptedObjectPipeline: """ cmm: AbstractCryptoMaterialsManager = field() + commitment_policy: CommitmentPolicy = field() s3_client: object = field(default=None) + enable_legacy_unauthenticated_modes: bool = field(default=False) + + # Map content cipher metadata values to AlgorithmSuite + _CONTENT_CIPHER_TO_ALGORITHM_SUITE = { + "AES/CBC/PKCS5Padding": AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF, + "AES/GCM/NoPadding": AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + "115": AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + } + + def _determine_algorithm_suite(self, metadata) -> AlgorithmSuite: + """Determine the algorithm suite from object metadata. + + V1 objects are always CBC. + V2/V3 objects check x-amz-cek-alg / x-amz-c to determine the content algorithm. + """ + if metadata.is_v1_format(): + ##= specification/s3-encryption/data-format/content-metadata.md#algorithm-suite-and-message-format-version-compatibility + ##= type=citation + ##% Objects encrypted with ALG_AES_256_CBC_IV16_NO_KDF MAY use either the V1 or V2 message format version. + return AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF + + if metadata.is_v2_format(): + cek_alg = metadata.content_cipher + if cek_alg is None: + raise S3EncryptionClientError( + "V2 format object missing required x-amz-cek-alg metadata." + ) + suite = self._CONTENT_CIPHER_TO_ALGORITHM_SUITE.get(cek_alg) + if suite is None: + raise S3EncryptionClientError(f"Unknown content encryption algorithm: {cek_alg}") + return suite + + if metadata.is_v3_format(): + cek_alg = metadata.content_cipher_v3 + if cek_alg is None: + raise S3EncryptionClientError("V3 format object missing required x-amz-c metadata.") + suite = self._CONTENT_CIPHER_TO_ALGORITHM_SUITE.get(cek_alg) + if suite is None: + raise S3EncryptionClientError(f"Unknown content encryption algorithm: {cek_alg}") + return suite + + raise S3EncryptionClientError("Unable to determine S3 Encryption Client message format.") def decrypt( self, @@ -129,11 +254,38 @@ def decrypt( if self.s3_client is None: raise S3EncryptionClientError("s3_client required to fetch instruction file") + # TODO: we should validate that these parameters must be None + # when not in instruction file mode. if bucket is None or key is None: raise S3EncryptionClientError("Bucket and key required to fetch instruction file") instruction_key = key + instruction_suffix instruction_metadata = fetch_instruction_file(self.s3_client, bucket, instruction_key) + + ##= specification/s3-encryption/data-format/metadata-strategy.md#v3-instruction-files + ##= type=implementation + ##% - The V3 message format MUST NOT store the mapkey "x-amz-c" and its value in the Instruction File. + ##= specification/s3-encryption/data-format/metadata-strategy.md#v3-instruction-files + ##= type=implementation + ##% - The V3 message format MUST NOT store the mapkey "x-amz-d" and its value in the Instruction File. + ##= specification/s3-encryption/data-format/metadata-strategy.md#v3-instruction-files + ##= type=implementation + ##% - The V3 message format MUST NOT store the mapkey "x-amz-i" and its value in the Instruction File. + v3_object_metadata_exclusive_keys = { + ObjectMetadata.CONTENT_CIPHER_V3, + ObjectMetadata.KEY_COMMITMENT_V3, + ObjectMetadata.MESSAGE_ID_V3, + } + forbidden_keys_in_instruction = ( + set(instruction_metadata.keys()) & v3_object_metadata_exclusive_keys + ) + if forbidden_keys_in_instruction: + raise S3EncryptionClientError( + "Instruction file is tampered, instruction file contains object metadata " + f"exclusive mapkeys: {forbidden_keys_in_instruction}. " + f"bucket: {bucket}\n key:{key}\n instruction_file:{instruction_key}" + ) + instruction_metadata.update(encryption_metadata) metadata = ObjectMetadata.from_dict(instruction_metadata) ##= specification/s3-encryption/data-format/metadata-strategy.md#v1-v2-instruction-files @@ -152,6 +304,10 @@ def decrypt( "BUT Instruction File is being used. This is an illegal combination. " f"bucket: {bucket}\n key:{key}\n instruction_file:{instruction_key}" ) + + # Determine the algorithm suite from the metadata + algorithm_suite = self._determine_algorithm_suite(metadata) + # Determine which format we're dealing with and get decryption materials if metadata.is_v1_format(): dec_materials = self._decrypt_v1(metadata, encryption_context) @@ -164,58 +320,257 @@ def decrypt( "Unable to determine S3 Encryption Client message format." ) + dec_materials.algorithm_suite = algorithm_suite + ##= specification/s3-encryption/decryption.md#cbc-decryption - ##= type=TODO + ##= type=implementation ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is NOT enabled, ##% the S3EC MUST throw an error which details that client was ##% not configured to decrypt objects with ALG_AES_256_CBC_IV16_NO_KDF. + if ( + algorithm_suite.is_legacy and not self.enable_legacy_unauthenticated_modes + ): # noqa: SIM102 + ##= specification/s3-encryption/decryption.md#legacy-decryption + ##= type=implementation + ##% The S3EC MUST NOT decrypt objects encrypted using legacy unauthenticated algorithm suites + ##% unless specifically configured to do so. + ##= specification/s3-encryption/decryption.md#legacy-decryption + ##= type=implementation + ##% If the S3EC is not configured to enable legacy unauthenticated content decryption, + ##% the client MUST throw an exception when attempting to decrypt an object encrypted + ##% with a legacy unauthenticated algorithm suite. + raise S3EncryptionClientError( + "Cannot decrypt object encrypted with ALG_AES_256_CBC_IV16_NO_KDF. " + "The S3 Encryption Client is not configured to decrypt objects using " + "legacy unauthenticated algorithm suites. " + "Set enable_legacy_unauthenticated_modes=True to allow decryption " + "of objects encrypted with CBC." + ) + + ##= specification/s3-encryption/decryption.md#key-commitment + ##= type=implementation + ##% The S3EC MUST validate the algorithm suite used for decryption against the + ##% key commitment policy before attempting to decrypt the content ciphertext. + ##= specification/s3-encryption/decryption.md#key-commitment + ##= type=implementation + ##% If the commitment policy requires decryption using a committing algorithm suite, + ##% and the algorithm suite associated with the object does not support key commitment, + ##% then the S3EC MUST throw an exception. + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##= type=implementation + ##% When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, the S3EC MUST NOT allow decryption using algorithm suites which do not support key commitment. + if ( + self.commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + and not dec_materials.algorithm_suite.supports_key_commitment + ): + raise S3EncryptionClientError( + "Configuration conflict: cannot decrypt non-key-committing object " + "when commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT. " + "Use REQUIRE_ENCRYPT_ALLOW_DECRYPT or FORBID_ENCRYPT_ALLOW_DECRYPT " + "to allow decryption of non-committing objects." + ) - # Perform decryption - aesgcm = AESGCM(dec_materials.plaintext_data_key) - return aesgcm.decrypt(nonce=dec_materials.iv, data=encrypted_data, associated_data=None) + # The FORBID_ENCRYPT_ALLOW_DECRYPT and REQUIRE_ENCRYPT_ALLOW_DECRYPT policies + # allow decryption with non-committing algorithm suites — no additional check needed. + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##= type=implementation + ##% When the commitment policy is FORBID_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST allow decryption using algorithm suites which do not support key commitment. + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##= type=implementation + ##% When the commitment policy is REQUIRE_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST allow decryption using algorithm suites which do not support key commitment. + + # Perform decryption based on algorithm suite + match dec_materials.algorithm_suite: + case AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF: + return self._decrypt_cbc_content(dec_materials, encrypted_data) + case AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf + ##= type=implementation + ##% The client MUST NOT provide any AAD when encrypting with + ##% ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + aesgcm = AESGCM(dec_materials.plaintext_data_key) + return aesgcm.decrypt( + nonce=dec_materials.iv, data=encrypted_data, associated_data=None + ) + case AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + return self._decrypt_kc_gcm_content(dec_materials, encrypted_data, metadata) + case _: + raise S3EncryptionClientError("Unknown algorithm suite!") def _decrypt_v2(self, metadata, encryption_context) -> DecryptionMaterials: """Prepare V2 decryption materials.""" - iv_bytes = base64.b64decode(metadata.content_iv) - edk_bytes = base64.b64decode(metadata.encrypted_data_key_v2) + return self._decrypt_v1_v2( + iv_b64=metadata.content_iv, + edk_b64=metadata.encrypted_data_key_v2, + wrap_alg=metadata.encrypted_data_key_algorithm, + stored_context=metadata.encrypted_data_key_context or {}, + encryption_context=encryption_context, + ) + + def _decrypt_v1(self, metadata, encryption_context) -> DecryptionMaterials: + """Prepare V1 decryption materials.""" + return self._decrypt_v1_v2( + iv_b64=metadata.content_iv, + edk_b64=metadata.encrypted_data_key_v1, + wrap_alg=metadata.encrypted_data_key_algorithm, + stored_context=metadata.encrypted_data_key_context or {}, + encryption_context=encryption_context, + ) + + def _decrypt_v1_v2( + self, iv_b64, edk_b64, wrap_alg, stored_context, encryption_context + ) -> DecryptionMaterials: + """Shared logic for preparing V1/V2 decryption materials.""" + iv_bytes = base64.b64decode(iv_b64) + edk_bytes = base64.b64decode(edk_b64) encrypted_data_key = EncryptedDataKey( key_provider_id=b"S3Keyring", - key_provider_info=metadata.encrypted_data_key_algorithm, + key_provider_info=wrap_alg, encrypted_data_key=edk_bytes, ) dec_materials = DecryptionMaterials( iv=iv_bytes, encrypted_data_keys=[encrypted_data_key], - encryption_context_stored=metadata.encrypted_data_key_context or {}, + encryption_context_stored=stored_context, encryption_context_from_request=encryption_context, ) return self.cmm.decrypt_materials(dec_materials) - def _decrypt_v1(self, metadata, encryption_context) -> DecryptionMaterials: - """Prepare V1 decryption materials.""" - iv_bytes = base64.b64decode(metadata.content_iv) - edk_bytes = base64.b64decode(metadata.encrypted_data_key_v1) + def _decrypt_cbc_content(self, dec_materials, encrypted_data): + """Decrypt content encrypted with ALG_AES_256_CBC_IV16_NO_KDF.""" + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=implementation + ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and + ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is enabled, + ##% then the S3EC MUST create a cipher with AES in CBC Mode with PKCS5Padding or + ##% PKCS7Padding compatible padding for a 16-byte block cipher + ##% (example: for the Java JCE, this is "AES/CBC/PKCS5Padding"). + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=implementation + ##% If the cipher object cannot be created as described above, + ##% Decryption MUST fail. + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=implementation + ##% The error SHOULD detail why the cipher could not be initialized + ##% (such as CBC or PKCS5Padding is not supported by the underlying crypto provider). + try: + cipher = Cipher( + algorithms.AES(dec_materials.plaintext_data_key), + modes.CBC(dec_materials.iv), + ) + decryptor = cipher.decryptor() + padded_plaintext = decryptor.update(encrypted_data) + decryptor.finalize() + + # Remove PKCS7 padding (compatible with PKCS5Padding for 16-byte block ciphers) + unpadder = PKCS7(128).unpadder() + return unpadder.update(padded_plaintext) + unpadder.finalize() + except Exception as e: + raise S3EncryptionClientSecurityError( + f"Failed to decrypt CBC content: {e}. " + "Ensure the underlying crypto provider supports AES/CBC/PKCS7Padding." + ) from e + + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only + ##% The V3 format uses compression here such that each wrapping algorithm is represented by a two digit string. + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only + ##% - The wrapping algorithm value "02" MUST be translated to AES/GCM upon retrieval, and vice versa on write. + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only + ##% - The wrapping algorithm value "12" MUST be translated to kms+context upon retrieval, and vice versa on write. + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only + ##% - The wrapping algorithm value "22" MUST be translated to RSA-OAEP-SHA1 upon retrieval, and vice versa on write. + _V3_WRAP_ALG_MAP = { + "02": "AES/GCM", + "12": "kms+context", + "22": "RSA-OAEP-SHA1", + } + + def _decrypt_v3(self, metadata, encryption_context) -> DecryptionMaterials: + """Prepare V3 decryption materials.""" + edk_bytes = base64.b64decode(metadata.encrypted_data_key_v3) + + # Map V3 compressed wrapping algorithm to canonical key_provider_info + raw_wrap_alg = metadata.encrypted_data_key_algorithm_v3 or "12" + wrap_alg = self._V3_WRAP_ALG_MAP.get(raw_wrap_alg, raw_wrap_alg) encrypted_data_key = EncryptedDataKey( key_provider_id=b"S3Keyring", - key_provider_info=metadata.encrypted_data_key_algorithm, + key_provider_info=wrap_alg, encrypted_data_key=edk_bytes, ) + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only + ##% The Encryption Context value MUST be used for wrapping algorithm `kms+context` or `12`. + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only + ##% The Material Description MUST be used for wrapping algorithms `AES/GCM` (`02`) and `RSA-OAEP-SHA1` (`22`). + # For kms+context, the stored context comes from x-amz-t (encryption_context_v3). + # For AES/GCM and RSA-OAEP-SHA1, it comes from x-amz-m (mat_desc_v3). + stored_context = {} + if wrap_alg == "kms+context": + raw_ctx = metadata.encryption_context_v3 + else: + raw_ctx = metadata.mat_desc_v3 + + if raw_ctx is not None: + if isinstance(raw_ctx, dict): + stored_context = raw_ctx + elif isinstance(raw_ctx, str): + stored_context = json.loads(raw_ctx) + dec_materials = DecryptionMaterials( - iv=iv_bytes, encrypted_data_keys=[encrypted_data_key], - encryption_context_stored=metadata.encrypted_data_key_context or {}, + encryption_context_stored=stored_context, encryption_context_from_request=encryption_context, ) return self.cmm.decrypt_materials(dec_materials) - def _decrypt_v3(self, metadata, encryption_context) -> DecryptionMaterials: - """Prepare V3 decryption materials.""" - # TODO: Implement V3 decryption - raise NotImplementedError("V3 decryption not yet implemented") + def _decrypt_kc_gcm_content(self, dec_materials, encrypted_data, metadata): + """Decrypt content encrypted with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY. + + Performs HKDF key derivation, key commitment verification, and AES-GCM decryption. + """ + message_id = base64.b64decode(metadata.message_id_v3) + stored_commitment = base64.b64decode(metadata.key_commitment_v3) + + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key + ##= type=implementation + ##% The client MUST use HKDF to derive the key commitment value and the derived encrypting key as described in [Key Derivation](key-derivation.md). + derived_encryption_key, derived_commitment = derive_keys( + dec_materials.plaintext_data_key, message_id, dec_materials.algorithm_suite + ) + + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##= type=implementation + ##% When using an algorithm suite which supports key commitment, the client MUST verify + ##% that the [derived key commitment](./key-derivation.md#hkdf-operation) contains the + ##% same bytes as the stored key commitment retrieved from the stored object's metadata. + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##= type=implementation + ##% When using an algorithm suite which supports key commitment, the client MUST verify the key commitment values match before deriving + ##% the [derived encryption key](./key-derivation.md#hkdf-operation). + verify_commitment(stored_commitment, derived_commitment) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% When encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ##% the IV used in the AES-GCM content encryption/decryption MUST consist entirely of bytes with the value 0x01. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% The IV's total length MUST match the IV length defined by the algorithm suite. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% The client MUST initialize the cipher, or call an AES-GCM encryption API, with the derived encryption key, an IV containing only bytes with the value 0x01, + ##% and the tag length defined in the Algorithm Suite when encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. + aesgcm = AESGCM(derived_encryption_key) + return aesgcm.decrypt( + nonce=dec_materials.algorithm_suite.kc_gcm_iv, + data=encrypted_data, + associated_data=dec_materials.algorithm_suite.suite_id_bytes, + ) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java new file mode 100644 index 00000000..46df7de1 --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java @@ -0,0 +1,91 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.encryption.s3; + +import static software.amazon.encryption.s3.TestUtils.*; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.S3ECConfig; + +/** + * Key Commitment Policy — Encryption Failure Tests + * + * Per the specification (key-commitment.md): + * "When the commitment policy is REQUIRE_ENCRYPT_ALLOW_DECRYPT, + * the S3EC MUST only encrypt using an algorithm suite which supports key commitment." + * "When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + * the S3EC MUST only encrypt using an algorithm suite which supports key commitment." + * "When the commitment policy is FORBID_ENCRYPT_ALLOW_DECRYPT, + * the S3EC MUST NOT encrypt using an algorithm suite which supports key commitment." + * + * These tests verify that attempting to encrypt with an algorithm that conflicts + * with the commitment policy is rejected by the S3EC — either at client creation + * or at PutObject time. + * + * Currently scoped to Python V3 only. Other languages can be enabled by + * switching the MethodSource to a broader provider (e.g. improvedClientsForTest). + */ +@DisplayName("Key Commitment Policy — Encrypt Failures") +public class KeyCommitmentPolicyEncryptFailureTests { + + private static final KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + @ParameterizedTest(name = "{0}: REQUIRE_ENCRYPT_ALLOW_DECRYPT with non-committing GCM MUST fail to encrypt") + @MethodSource("software.amazon.encryption.s3.TestUtils#pythonV3ClientForTest") + void require_encrypt_allow_decrypt_with_non_committing_gcm_must_fail( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + S3ECConfig config = S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build(); + + TestUtils.Encrypt_fails(client, config, + appendTestSuffix("test-kc-policy-fail-REAC-gcm-" + language.getLanguageName())); + } + + @ParameterizedTest(name = "{0}: REQUIRE_ENCRYPT_REQUIRE_DECRYPT with non-committing GCM MUST fail to encrypt") + @MethodSource("software.amazon.encryption.s3.TestUtils#pythonV3ClientForTest") + void require_encrypt_require_decrypt_with_non_committing_gcm_must_fail( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + S3ECConfig config = S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build(); + + TestUtils.Encrypt_fails(client, config, + appendTestSuffix("test-kc-policy-fail-RERD-gcm-" + language.getLanguageName())); + } + + @ParameterizedTest(name = "{0}: FORBID_ENCRYPT_ALLOW_DECRYPT with committing GCM MUST fail to encrypt") + @MethodSource("software.amazon.encryption.s3.TestUtils#pythonV3ClientForTest") + void forbid_encrypt_allow_decrypt_with_committing_gcm_must_fail( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + S3ECConfig config = S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + .build(); + + TestUtils.Encrypt_fails(client, config, + appendTestSuffix("test-kc-policy-fail-FEAD-kc-gcm-" + language.getLanguageName())); + } +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index 2b9cd062..d488fd2d 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -35,12 +35,15 @@ import org.junit.jupiter.params.provider.Arguments; import com.amazonaws.regions.Region; import com.amazonaws.regions.Regions; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; import software.amazon.encryption.s3.model.EncryptionAlgorithm; import software.amazon.encryption.s3.model.GetObjectInput; import software.amazon.encryption.s3.model.GetObjectOutput; import software.amazon.encryption.s3.model.KeyMaterial; import software.amazon.encryption.s3.model.PutObjectInput; import software.amazon.encryption.s3.model.PutObjectOutput; +import software.amazon.encryption.s3.model.S3ECConfig; import software.amazon.encryption.s3.model.S3EncryptionClientError; import software.amazon.smithy.java.aws.client.restjson.RestJsonClientProtocol; import software.amazon.smithy.java.client.core.ClientConfig; @@ -155,7 +158,7 @@ public class TestUtils { public static final Set IMPROVED_VERSIONS = Set.of( JAVA_V4, - // PYTHON_V3, + PYTHON_V3, GO_V4, NET_V4, CPP_V3, @@ -358,6 +361,17 @@ public static Stream improvedClientsForTest() { .map(Arguments::of); } + /** + * Get stream of arguments for the Python V3 client only. + * Other languages can be added to this set as their commitment policy + * validation is confirmed. + */ + public static Stream pythonV3ClientForTest() { + return serverMap.values().stream() + .filter(target -> PYTHON_V3.equals(target.getLanguageName())) + .map(Arguments::of); + } + /** * Get stream of arguments for clients that support RAW AES (includes CPP). */ @@ -666,6 +680,40 @@ public static void DecryptWithMaterialsDescription( } } + /** + * Attempts to encrypt an object and expects the operation to fail with an S3EncryptionClientError. + * This is used for negative tests where the client configuration should prevent encryption + * (e.g., commitment policy violations). + * + * The failure may occur during client creation (CreateClient) or during the PutObject call, + * depending on when the server-side S3EC validates the configuration. + */ + public static void Encrypt_fails( + S3ECTestServerClient client, + S3ECConfig config, + String objectKey + ) { + try { + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(config) + .build()); + String S3ECId = clientOutput.getClientId(); + + client.putObject(PutObjectInput.builder() + .clientID(S3ECId) + .key(objectKey) + .bucket(TestUtils.BUCKET) + .body(ByteBuffer.wrap(objectKey.getBytes(StandardCharsets.UTF_8))) + .build()); + + fail("Encryption should have failed for object: " + objectKey + + " with config commitmentPolicy=" + config.getCommitmentPolicy() + + " encryptionAlgorithm=" + config.getEncryptionAlgorithm()); + } catch (S3EncryptionClientError e) { + // Expected - the S3EC should reject this configuration + } + } + public static void Decrypt_fails( S3ECTestServerClient client, String S3ECId, List crossLanguageObjects, diff --git a/test-server/python-v3-server/src/main.py b/test-server/python-v3-server/src/main.py index 6c57c6bd..cee2ab4e 100755 --- a/test-server/python-v3-server/src/main.py +++ b/test-server/python-v3-server/src/main.py @@ -7,6 +7,7 @@ from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.exceptions import S3EncryptionClientError from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy import boto3 import uvicorn import json @@ -67,6 +68,19 @@ def create_s3_encryption_client_error( ) +# Maps from Smithy model enum strings to Python AlgorithmSuite/CommitmentPolicy enums +_ALGORITHM_SUITE_MAP = { + "ALG_AES_256_GCM_IV12_TAG16_NO_KDF": AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY": AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, +} + +_COMMITMENT_POLICY_MAP = { + "FORBID_ENCRYPT_ALLOW_DECRYPT": CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + "REQUIRE_ENCRYPT_ALLOW_DECRYPT": CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + "REQUIRE_ENCRYPT_REQUIRE_DECRYPT": CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, +} + + @app.put("/object/{bucket}/{key}") async def put_object(bucket: str, key: str, request: Request): """ @@ -175,6 +189,7 @@ async def client_endpoint(request: Request): key_material = config_data.get("keyMaterial", {}) enable_legacy_wrapping_algorithms = config_data.get("enableLegacyWrappingAlgorithms", False) + enable_legacy_unauthenticated_modes = config_data.get("enableLegacyUnauthenticatedModes", False) # TODO pull region from ARN kms_client = boto3.client("kms", region_name="us-west-2") @@ -185,7 +200,26 @@ async def client_endpoint(request: Request): enable_legacy_wrapping_algorithms=enable_legacy_wrapping_algorithms, ) wrapped_client = boto3.client("s3") - client_config = S3EncryptionClientConfig(keyring) + + # Build config kwargs, only including algorithm_suite and commitment_policy if provided + config_kwargs = { + "keyring": keyring, + "enable_legacy_unauthenticated_modes": enable_legacy_unauthenticated_modes, + } + + encryption_algorithm = config_data.get("encryptionAlgorithm") + if encryption_algorithm is not None: + if encryption_algorithm not in _ALGORITHM_SUITE_MAP: + raise ValueError(f"Unknown encryption algorithm: {encryption_algorithm}") + config_kwargs["encryption_algorithm"] = _ALGORITHM_SUITE_MAP[encryption_algorithm] + + commitment_policy = config_data.get("commitmentPolicy") + if commitment_policy is not None: + if commitment_policy not in _COMMITMENT_POLICY_MAP: + raise ValueError(f"Unknown commitment policy: {commitment_policy}") + config_kwargs["commitment_policy"] = _COMMITMENT_POLICY_MAP[commitment_policy] + + client_config = S3EncryptionClientConfig(**config_kwargs) # Create S3EncryptionClient client = S3EncryptionClient(wrapped_client, client_config) diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index 616f8da4..15133c05 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -9,6 +9,7 @@ from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.exceptions import S3EncryptionClientError from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") region = os.environ.get("CI_AWS_REGION", "us-west-2") @@ -16,457 +17,238 @@ "CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" ) - -def test_simple_roundtrip_ascii_string(): - key = "simple-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - - data = "test input for simple v3 round trip" - +# Parameterized algorithm suite configurations. +# Each entry is (algorithm_suite, commitment_policy, id_label). +# "default" uses the client defaults (KC GCM + Require/Require). +ALGORITHM_CONFIGS = [ + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + id="AES_GCM", + ), + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + id="KC_GCM", + ), +] + + +def _make_client(algorithm_suite, commitment_policy): + """Create an S3EncryptionClient with the given algorithm config.""" kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - s3ec.put_object(Bucket=bucket, Key=key, Body=data) - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) - output = response["Body"].read().decode("utf-8") - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(input) - print("Output:") - print(output) - raise RuntimeError - print("Success!") - + config = S3EncryptionClientConfig( + keyring, + encryption_algorithm=algorithm_suite, + commitment_policy=commitment_policy, + ) + return S3EncryptionClient(wrapped_client, config) -def test_empty_string_roundtrip(): - key = "empty-string-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - data = "" # Empty string as test data +def _unique_key(prefix): + """Generate a unique S3 key with a timestamp suffix.""" + return prefix + datetime.now().strftime("%Y-%m-%d-%H:%M:%S-%f") - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_simple_roundtrip_ascii_string(algorithm_suite, commitment_policy): + key = _unique_key("simple-rt-") + data = "test input for simple v3 round trip" - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data) - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) + response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read().decode("utf-8") - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) # Using repr to clearly show it's an empty string - print("Output:") - print(repr(output)) - raise RuntimeError - print("Success! Empty string encrypted and decrypted correctly.") + assert output == data -def test_no_body_roundtrip(): - key = "no-body-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_empty_string_roundtrip(algorithm_suite, commitment_policy): + key = _unique_key("empty-string-rt-") + data = "" - # Expected data when no Body is provided (empty bytes) - expected_data = b"" + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + response = s3ec.get_object(Bucket=bucket, Key=key) + output = response["Body"].read().decode("utf-8") + assert output == data - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_no_body_roundtrip(algorithm_suite, commitment_policy): + key = _unique_key("no-body-rt-") + expected_data = b"" - # Call put_object without providing a Body parameter + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key) - - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) + response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read() + assert output == expected_data - if output != expected_data: - print("Uh oh! Output doesn't match expected empty bytes!") - print("Expected:") - print(repr(expected_data)) - print("Output:") - print(repr(output)) - raise RuntimeError - print( - "Success! Object with no Body parameter encrypted and decrypted correctly as empty bytes." - ) - - -def test_unicode_string_roundtrip(): - key = "unicode-string-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - # String with unusual Unicode characters +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_unicode_string_roundtrip(algorithm_suite, commitment_policy): + key = _unique_key("unicode-string-rt-") data = "Unicode test: 你好, こんにちは, 안녕하세요, Привет, مرحبا, ¡Hola!, ½⅓¼⅕⅙⅐⅛⅑⅒⅔⅖⅗⅘⅙⅚⅜⅝⅞" - kms_client = boto3.client("kms", region_name=region) - - keyring = KmsKeyring(kms_client, kms_key_id) - - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data) - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) - - # Boto3 encodes to utf-8 in put_object but does not - # decode in get_object; do so manually to complete the - # round trip + response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read().decode("utf-8") - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) - print("Output:") - print(repr(output)) - raise RuntimeError - print("Success! Unicode string encrypted and decrypted correctly.") - + assert output == data -def test_specific_encoding_utf8_roundtrip(): - key = "utf8-encoding-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - # String with mixed characters +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_specific_encoding_utf8_roundtrip(algorithm_suite, commitment_policy): + key = _unique_key("utf8-encoding-rt-") data = "UTF-8 encoding test: 你好, こんにちは, 안녕하세요, Привет, مرحبا, ¡Hola!" - - # Explicitly encode as UTF-8 before sending encoded_data = data.encode("utf-8") - kms_client = boto3.client("kms", region_name=region) - - keyring = KmsKeyring(kms_client, kms_key_id) - - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Pass the pre-encoded bytes to put_object + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=encoded_data) - - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) - - # Read raw bytes and decode with the same encoding + response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read().decode("utf-8") + assert output == data - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) - print("Output:") - print(repr(output)) - raise RuntimeError - print("Success! UTF-8 encoded string encrypted and decrypted correctly.") - - -def test_specific_encoding_latin1_roundtrip(): - key = "latin1-encoding-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - # String with Latin-1 compatible characters +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_specific_encoding_latin1_roundtrip(algorithm_suite, commitment_policy): + key = _unique_key("latin1-encoding-rt-") data = "Latin-1 encoding test: éèêë àâäãåá çñ ¿¡ øæå ØÆÅÉÈÊËÀÂÄÃÅÁ" - - # Explicitly encode as Latin-1 before sending encoded_data = data.encode("latin-1") - kms_client = boto3.client("kms", region_name=region) - - keyring = KmsKeyring(kms_client, kms_key_id) - - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Pass the pre-encoded bytes to put_object + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=encoded_data) - - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) - - # Read raw bytes and decode with the same encoding + response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read().decode("latin-1") + assert output == data - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) - print("Output:") - print(repr(output)) - raise RuntimeError - print("Success! Latin-1 encoded string encrypted and decrypted correctly.") - - -def test_binary_data_roundtrip(): - key = "binary-data-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - # Create some binary data (not valid in any particular encoding) +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_binary_data_roundtrip(algorithm_suite, commitment_policy): + key = _unique_key("binary-data-rt-") data = bytes(range(256)) - kms_client = boto3.client("kms", region_name=region) - - keyring = KmsKeyring(kms_client, kms_key_id) - - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Pass the binary data directly + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data) - - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) - - # Read raw bytes without decoding + response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read() - - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) - print("Output:") - print(repr(output)) - raise RuntimeError - print("Success! Binary data encrypted and decrypted correctly.") + assert output == data -def test_invalid_body_types(): +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_invalid_body_types(algorithm_suite, commitment_policy): """Test that put_object raises an exception when given invalid body types.""" - key = "invalid-body-type" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Test with integer - with pytest.raises(S3EncryptionClientError) as excinfo: - s3ec.put_object(Bucket=bucket, Key=key, Body=42) - assert "Invalid type for parameter Body" in str(excinfo.value) - - # Test with float - with pytest.raises(S3EncryptionClientError) as excinfo: - s3ec.put_object(Bucket=bucket, Key=key, Body=3.14) - assert "Invalid type for parameter Body" in str(excinfo.value) - - # Test with list - with pytest.raises(S3EncryptionClientError) as excinfo: - s3ec.put_object(Bucket=bucket, Key=key, Body=[1, 2, 3]) - assert "Invalid type for parameter Body" in str(excinfo.value) - - # Test with dictionary - with pytest.raises(S3EncryptionClientError) as excinfo: - s3ec.put_object(Bucket=bucket, Key=key, Body={"key": "value"}) - assert "Invalid type for parameter Body" in str(excinfo.value) - - # Test with boolean - with pytest.raises(S3EncryptionClientError) as excinfo: - s3ec.put_object(Bucket=bucket, Key=key, Body=True) - assert "Invalid type for parameter Body" in str(excinfo.value) + key = _unique_key("invalid-body-type-") - # Test with None (also raises an exception) - with pytest.raises(S3EncryptionClientError) as excinfo: - s3ec.put_object(Bucket=bucket, Key=key, Body=None) - assert "Invalid type for parameter Body" in str(excinfo.value) + s3ec = _make_client(algorithm_suite, commitment_policy) - print("Success! All invalid body types correctly raised exceptions.") + for body in [42, 3.14, [1, 2, 3], {"key": "value"}, True, None]: + with pytest.raises(S3EncryptionClientError) as excinfo: + s3ec.put_object(Bucket=bucket, Key=key, Body=body) + assert "Invalid type for parameter Body" in str(excinfo.value) -def test_user_metadata_preservation(): +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_user_metadata_preservation(algorithm_suite, commitment_policy): """Test that user-provided metadata is preserved during encryption.""" - key = "metadata-preservation-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - + key = _unique_key("metadata-preservation-rt-") data = "Test data with user metadata" - - # User metadata to include user_metadata = { "author": "test-user", "version": "1.0", "description": "Test object with custom metadata", } - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Put object with user metadata + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data, Metadata=user_metadata) + response = s3ec.get_object(Bucket=bucket, Key=key) - # Get the object back - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) - - # Verify the data decrypts correctly output = response["Body"].read().decode("utf-8") - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) - print("Output:") - print(repr(output)) - raise RuntimeError - - # Verify user metadata is preserved - returned_metadata = response.get("Metadata", {}) + assert output == data + returned_metadata = response.get("Metadata", {}) for key_name, expected_value in user_metadata.items(): - if key_name not in returned_metadata: - print(f"Uh oh! User metadata key '{key_name}' is missing!") - print("Expected metadata:") - print(user_metadata) - print("Returned metadata:") - print(returned_metadata) - raise RuntimeError - - if returned_metadata[key_name] != expected_value: - print(f"Uh oh! User metadata value for '{key_name}' doesn't match!") - print(f"Expected: {expected_value}") - print(f"Got: {returned_metadata[key_name]}") - raise RuntimeError - - print("Success! User metadata preserved correctly during encryption/decryption.") - print(f"User metadata: {user_metadata}") - print(f"Returned metadata keys: {list(returned_metadata.keys())}") - - -def test_encryption_context_roundtrip(): - """Test that EncryptionContext is properly used during encryption and required for decryption.""" - key = "encryption-context-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + assert key_name in returned_metadata, f"User metadata key '{key_name}' is missing" + assert returned_metadata[key_name] == expected_value - data = "Test data with encryption context" - # Encryption context to use for additional authenticated data +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_encryption_context_roundtrip(algorithm_suite, commitment_policy): + """Test that EncryptionContext is properly used during encryption and required for decryption.""" + key = _unique_key("encryption-context-rt-") + data = "Test data with encryption context" encryption_context = { "department": "engineering", "project": "s3-encryption", "environment": "test", } - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Put object with encryption context + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) + response = s3ec.get_object(Bucket=bucket, Key=key, EncryptionContext=encryption_context) - # Get the object back WITH the same encryption context - get_req = {"Bucket": bucket, "Key": key, "EncryptionContext": encryption_context} - response = s3ec.get_object(**get_req) - - # Verify the data decrypts correctly output = response["Body"].read().decode("utf-8") - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) - print("Output:") - print(repr(output)) - raise RuntimeError + assert output == data - print("Success! Encryption context used correctly during encryption/decryption.") - print(f"Encryption context: {encryption_context}") - -def test_encryption_context_mismatch(): +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_encryption_context_mismatch(algorithm_suite, commitment_policy): """Test that decryption fails when EncryptionContext doesn't match.""" - key = "encryption-context-mismatch" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - + key = _unique_key("encryption-context-mismatch-") data = "Test data with encryption context" - - # Original encryption context encryption_context = {"department": "engineering", "project": "s3-encryption"} - - # Wrong encryption context for decryption wrong_encryption_context = {"department": "marketing", "project": "s3-encryption"} - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Put object with encryption context + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) - # Try to get the object back with WRONG encryption context - should fail - get_req = {"Bucket": bucket, "Key": key, "EncryptionContext": wrong_encryption_context} - - try: - s3ec.get_object(**get_req) - # If we get here, the test failed - decryption should have failed - print("Uh oh! Decryption succeeded with wrong encryption context!") - print(f"Original context: {encryption_context}") - print(f"Wrong context used: {wrong_encryption_context}") - raise RuntimeError("Expected decryption to fail with mismatched encryption context") - except S3EncryptionClientError as e: - # This is expected - decryption should fail - print("Success! Decryption correctly failed with mismatched encryption context.") - print(f"Error message: {str(e)}") - except Exception as e: - # Some other error occurred - print(f"Unexpected error type: {type(e).__name__}") - print(f"Error message: {str(e)}") - raise - - -def test_encryption_context_missing_on_decrypt(): - """Test that decryption fails when encryption context is not provided for an object encrypted with context.""" - key = "encryption-context-missing" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + with pytest.raises(S3EncryptionClientError): + s3ec.get_object(Bucket=bucket, Key=key, EncryptionContext=wrong_encryption_context) - data = "Test data with encryption context" - # Encryption context used during encryption +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_encryption_context_missing_on_decrypt(algorithm_suite, commitment_policy): + """Test that decryption fails when encryption context is not provided for an object encrypted with context.""" + key = _unique_key("encryption-context-missing-") + data = "Test data with encryption context" encryption_context = {"department": "engineering", "project": "s3-encryption"} - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Put object with encryption context + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) - # Try to get the object back WITHOUT encryption context - should fail - get_req = {"Bucket": bucket, "Key": key} - - try: - s3ec.get_object(**get_req) - # If we get here, the test failed - decryption should have failed - print("Uh oh! Decryption succeeded without providing required encryption context!") - print(f"Original context: {encryption_context}") - raise RuntimeError("Expected decryption to fail when encryption context not provided") - except S3EncryptionClientError as e: - # This is expected - decryption should fail - print("Success! Decryption correctly failed when encryption context was not provided.") - print(f"Error message: {str(e)}") - except Exception as e: - # Some other error occurred - print(f"Unexpected error type: {type(e).__name__}") - print(f"Error message: {str(e)}") - raise + with pytest.raises(S3EncryptionClientError): + s3ec.get_object(Bucket=bucket, Key=key) + + +# Expected metadata key that identifies the content encryption algorithm, +# keyed by algorithm suite. +_EXPECTED_ALGORITHM_METADATA = { + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: ("x-amz-cek-alg", "AES/GCM/NoPadding"), + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: ("x-amz-c", "115"), +} + + +##= specification/s3-encryption/encryption.md#content-encryption +##= type=test +##% The S3EC MUST use the encryption algorithm configured during +##% [client](./client.md) initialization. +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_put_object_uses_configured_algorithm(algorithm_suite, commitment_policy): + """PutObject MUST encrypt using the algorithm suite configured at client init.""" + key = _unique_key("configured-alg-") + data = b"test configured algorithm" + + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + + # Read back with a plain S3 client to inspect the raw metadata + plain_s3 = boto3.client("s3") + response = plain_s3.head_object(Bucket=bucket, Key=key) + metadata = response.get("Metadata", {}) + + meta_key, expected_value = _EXPECTED_ALGORITHM_METADATA[algorithm_suite] + assert meta_key in metadata, f"Expected metadata key '{meta_key}' not found in {metadata}" + assert metadata[meta_key] == expected_value diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index 467ddc7a..f4f70704 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -7,6 +7,7 @@ from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy # Static test objects bucket bucket = os.environ.get("CI_S3_STATIC_TEST_BUCKET", "s3ec-static-test-objects") @@ -26,8 +27,6 @@ } -# TODO(cbc): enable once CBC decryption is implemented -@pytest.mark.skip(reason="V1 CBC decryption not yet implemented") def test_decrypt_v1_instruction_file(): """Test decrypting V1 object with instruction file. @@ -39,7 +38,12 @@ def test_decrypt_v1_instruction_file(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id, enable_legacy_wrapping_algorithms=True) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig( + keyring, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + enable_legacy_unauthenticated_modes=True, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) response = s3ec.get_object(Bucket=bucket, Key=key) @@ -60,7 +64,11 @@ def test_decrypt_v2_instruction_file(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig( + keyring, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) response = s3ec.get_object(Bucket=bucket, Key=key) @@ -70,8 +78,6 @@ def test_decrypt_v2_instruction_file(): print("Success! V2 instruction file decryption completed.") -# TODO(v3): enable once v3 is implemented -@pytest.mark.skip(reason="V3 decryption not yet implemented") def test_decrypt_v3_instruction_file(): """Test decrypting V3 object with instruction file. @@ -83,13 +89,16 @@ def test_decrypt_v3_instruction_file(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig( + keyring, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read().decode("utf-8") - assert output != "static-v3-instruction-file-from-java-v4" + assert output == "static-v3-instruction-file-from-java-v4" print("Success! V3 instruction file decryption completed.") @@ -115,8 +124,6 @@ def test_decrypt_invalid_instruction_file(): print(f"Error message: {exc_info.value}") -# TODO(v3): enable once v3 is implemented -@pytest.mark.skip(reason="V3 decryption not yet implemented") def test_decrypt_v3_instruction_file_custom_suffix(): """Test decrypting V3 object with a custom instruction file suffix.""" key = TEST_OBJECTS["v3_instruction_file"] @@ -124,7 +131,11 @@ def test_decrypt_v3_instruction_file_custom_suffix(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring, instruction_file_suffix=".custom-suffix-instruction") + config = S3EncryptionClientConfig( + keyring, + instruction_file_suffix=".custom-suffix-instruction", + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) response = s3ec.get_object(Bucket=bucket, Key=key) @@ -141,7 +152,12 @@ def test_decrypt_v2_instruction_file_custom_suffix(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring, instruction_file_suffix=".custom-suffix-instruction") + config = S3EncryptionClientConfig( + keyring, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + instruction_file_suffix=".custom-suffix-instruction", + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) response = s3ec.get_object(Bucket=bucket, Key=key) diff --git a/test/integration/test_i_s3_encryption_multithreaded.py b/test/integration/test_i_s3_encryption_multithreaded.py index 419ca7ea..e71a17df 100644 --- a/test/integration/test_i_s3_encryption_multithreaded.py +++ b/test/integration/test_i_s3_encryption_multithreaded.py @@ -16,6 +16,7 @@ from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.exceptions import S3EncryptionClientError from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") region = os.environ.get("CI_AWS_REGION", "us-west-2") @@ -38,7 +39,11 @@ def test_multithreaded_encryption_context_isolation(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig( + keyring, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) # Number of threads to test with @@ -150,7 +155,11 @@ def test_multithreaded_rapid_context_switching(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig( + keyring, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) num_iterations = 20 @@ -228,7 +237,11 @@ def test_multithreaded_mixed_with_and_without_context(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig( + keyring, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) errors = [] diff --git a/test/test_decryption.py b/test/test_decryption.py new file mode 100644 index 00000000..6d22d439 --- /dev/null +++ b/test/test_decryption.py @@ -0,0 +1,390 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for decryption specification compliance annotations. + +Each test in this module corresponds to a MUST/SHOULD requirement from +specification/s3-encryption/decryption.md and carries a type=test annotation +that mirrors the type=implementation annotation in the source code. +""" + +import base64 +import os +from io import BytesIO +from unittest.mock import Mock + +import pytest + +from s3_encryption.exceptions import S3EncryptionClientError, S3EncryptionClientSecurityError +from s3_encryption.key_derivation import derive_keys, verify_commitment +from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager +from s3_encryption.materials.keyring import S3Keyring +from s3_encryption.materials.materials import ( + AlgorithmSuite, + CommitmentPolicy, + DecryptionMaterials, +) +from s3_encryption.pipelines import GetEncryptedObjectPipeline + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_pipeline( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + enable_legacy=False, + s3_client=None, + keyring_side_effect=None, + keyring_return=None, +): + """Create a GetEncryptedObjectPipeline with a mocked CMM/keyring.""" + mock_keyring = Mock(spec=S3Keyring) + if keyring_side_effect is not None: + mock_keyring.on_decrypt.side_effect = keyring_side_effect + elif keyring_return is not None: + mock_keyring.on_decrypt.return_value = keyring_return + cmm = DefaultCryptoMaterialsManager(mock_keyring) + return GetEncryptedObjectPipeline( + cmm, + s3_client=s3_client, + commitment_policy=commitment_policy, + enable_legacy_unauthenticated_modes=enable_legacy, + ) + + +def _v1_cbc_metadata(): + """Return V1 (CBC) object metadata dict.""" + return { + "x-amz-iv": base64.b64encode(os.urandom(16)).decode(), + "x-amz-key": base64.b64encode(b"encrypted-key").decode(), + "x-amz-matdesc": '{"kms_cmk_id": "key-id"}', + } + + +def _v2_gcm_metadata(): + """Return V2 (GCM, no KDF) object metadata dict.""" + return { + "x-amz-iv": base64.b64encode(os.urandom(12)).decode(), + "x-amz-key-v2": base64.b64encode(b"encrypted-key").decode(), + "x-amz-wrap-alg": "kms+context", + "x-amz-matdesc": "{}", + "x-amz-cek-alg": "AES/GCM/NoPadding", + "x-amz-tag-len": "128", + } + + +def _response(metadata, body=b"ciphertext"): + return {"Body": BytesIO(body), "Metadata": metadata} + + +# --------------------------------------------------------------------------- +# CBC Decryption +# --------------------------------------------------------------------------- + + +class TestCBCDecryption: + """Tests for specification/s3-encryption/decryption.md#cbc-decryption.""" + + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=test + ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and + ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is NOT enabled, + ##% the S3EC MUST throw an error which details that client was + ##% not configured to decrypt objects with ALG_AES_256_CBC_IV16_NO_KDF. + def test_cbc_object_rejected_when_legacy_disabled(self): + """CBC-encrypted objects MUST be rejected when legacy modes are disabled.""" + plaintext_key = os.urandom(32) + dec_mats = DecryptionMaterials( + iv=os.urandom(16), + plaintext_data_key=plaintext_key, + algorithm_suite=AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF, + ) + pipeline = _make_pipeline( + enable_legacy=False, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + keyring_return=dec_mats, + ) + + with pytest.raises(S3EncryptionClientError, match="ALG_AES_256_CBC_IV16_NO_KDF"): + pipeline.decrypt(_response(_v1_cbc_metadata())) + + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=test + ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and + ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is enabled, + ##% then the S3EC MUST create a cipher with AES in CBC Mode with PKCS5Padding or + ##% PKCS7Padding compatible padding for a 16-byte block cipher + ##% (example: for the Java JCE, this is "AES/CBC/PKCS5Padding"). + def test_cbc_decryption_succeeds_when_legacy_enabled(self): + """CBC decryption MUST work with PKCS7-compatible padding when legacy is enabled.""" + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.primitives.padding import PKCS7 + + plaintext = b"hello world, this is a CBC test!!" + key = os.urandom(32) + iv = os.urandom(16) + + # Encrypt with AES-CBC + PKCS7 padding + padder = PKCS7(128).padder() + padded = padder.update(plaintext) + padder.finalize() + cipher = Cipher(algorithms.AES(key), modes.CBC(iv)) + encryptor = cipher.encryptor() + ciphertext = encryptor.update(padded) + encryptor.finalize() + + metadata = { + "x-amz-iv": base64.b64encode(iv).decode(), + "x-amz-key": base64.b64encode(b"encrypted-key").decode(), + "x-amz-matdesc": '{"kms_cmk_id": "key-id"}', + } + + dec_mats = DecryptionMaterials( + iv=iv, + plaintext_data_key=key, + algorithm_suite=AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF, + ) + pipeline = _make_pipeline( + enable_legacy=True, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + keyring_return=dec_mats, + ) + + result = pipeline.decrypt(_response(metadata, ciphertext)) + assert result == plaintext + + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=test + ##% If the cipher object cannot be created as described above, + ##% Decryption MUST fail. + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=test + ##% The error SHOULD detail why the cipher could not be initialized + ##% (such as CBC or PKCS5Padding is not supported by the underlying crypto provider). + def test_cbc_decryption_fails_with_wrong_key(self): + """CBC decryption MUST fail (with detail) when the cipher cannot decrypt.""" + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.primitives.padding import PKCS7 + + plaintext = b"hello world, this is a CBC test!!" + real_key = os.urandom(32) + wrong_key = os.urandom(32) + iv = os.urandom(16) + + padder = PKCS7(128).padder() + padded = padder.update(plaintext) + padder.finalize() + cipher = Cipher(algorithms.AES(real_key), modes.CBC(iv)) + encryptor = cipher.encryptor() + ciphertext = encryptor.update(padded) + encryptor.finalize() + + metadata = { + "x-amz-iv": base64.b64encode(iv).decode(), + "x-amz-key": base64.b64encode(b"encrypted-key").decode(), + "x-amz-matdesc": '{"kms_cmk_id": "key-id"}', + } + + dec_mats = DecryptionMaterials( + iv=iv, + plaintext_data_key=wrong_key, + algorithm_suite=AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF, + ) + pipeline = _make_pipeline( + enable_legacy=True, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + keyring_return=dec_mats, + ) + + with pytest.raises(S3EncryptionClientSecurityError, match="Failed to decrypt CBC content"): + pipeline.decrypt(_response(metadata, ciphertext)) + + +# --------------------------------------------------------------------------- +# Decrypting with Commitment +# --------------------------------------------------------------------------- + + +class TestDecryptingWithCommitment: + """Tests for specification/s3-encryption/decryption.md#decrypting-with-commitment.""" + + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##= type=test + ##% When using an algorithm suite which supports key commitment, the client MUST verify + ##% that the [derived key commitment](./key-derivation.md#hkdf-operation) contains the + ##% same bytes as the stored key commitment retrieved from the stored object's metadata. + def test_commitment_verified_against_stored_metadata(self): + """The derived commitment MUST match the stored commitment from metadata.""" + key = os.urandom(32) + message_id = os.urandom(28) + _, correct_commitment = derive_keys( + key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) + + # Should not raise + verify_commitment(correct_commitment, correct_commitment) + + # Tampered commitment must fail + tampered = bytearray(correct_commitment) + tampered[0] ^= 0xFF + with pytest.raises(S3EncryptionClientSecurityError): + verify_commitment(bytes(tampered), correct_commitment) + + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##= type=test + ##% When using an algorithm suite which supports key commitment, the verification of the derived key commitment value MUST be done in constant time. + def test_commitment_verification_uses_constant_time_compare(self): + """Verification MUST use constant-time comparison (hmac.compare_digest).""" + stored = os.urandom(28) + derived = os.urandom(28) + + # verify_commitment delegates to hmac.compare_digest; confirm it raises + # on mismatch (the constant-time property is guaranteed by hmac.compare_digest). + with pytest.raises(S3EncryptionClientSecurityError): + verify_commitment(stored, derived) + + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##= type=test + ##% When using an algorithm suite which supports key commitment, the client MUST throw an exception when the derived key commitment value + ##% and stored key commitment value do not match. + def test_commitment_mismatch_throws_exception(self): + """Mismatched commitment values MUST raise an exception.""" + stored = os.urandom(28) + derived = os.urandom(28) + + with pytest.raises( + S3EncryptionClientSecurityError, match="Key commitment verification failed" + ): + verify_commitment(stored, derived) + + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##= type=test + ##% When using an algorithm suite which supports key commitment, the client MUST verify the key commitment values match before deriving + ##% the [derived encryption key](./key-derivation.md#hkdf-operation). + def test_commitment_verified_before_content_decryption(self): + """Commitment verification MUST happen before content decryption is attempted.""" + key = os.urandom(32) + message_id = os.urandom(28) + _, real_commitment = derive_keys( + key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) + + # Build V3 metadata with a wrong commitment + wrong_commitment = os.urandom(28) + metadata = { + "x-amz-c": "115", + "x-amz-3": base64.b64encode(b"encrypted-key").decode(), + "x-amz-w": "12", + "x-amz-t": "{}", + "x-amz-d": base64.b64encode(wrong_commitment).decode(), + "x-amz-i": base64.b64encode(message_id).decode(), + } + + dec_mats = DecryptionMaterials( + plaintext_data_key=key, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ) + pipeline = _make_pipeline( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + keyring_return=dec_mats, + ) + + # Must fail at commitment check, not at AES-GCM decryption + with pytest.raises( + S3EncryptionClientSecurityError, match="Key commitment verification failed" + ): + pipeline.decrypt(_response(metadata, b"fake-ciphertext")) + + +# --------------------------------------------------------------------------- +# Key Commitment Policy +# --------------------------------------------------------------------------- + + +class TestKeyCommitmentPolicy: + """Tests for specification/s3-encryption/decryption.md#key-commitment.""" + + ##= specification/s3-encryption/decryption.md#key-commitment + ##= type=test + ##% The S3EC MUST validate the algorithm suite used for decryption against the + ##% key commitment policy before attempting to decrypt the content ciphertext. + ##= specification/s3-encryption/decryption.md#key-commitment + ##= type=test + ##% If the commitment policy requires decryption using a committing algorithm suite, + ##% and the algorithm suite associated with the object does not support key commitment, + ##% then the S3EC MUST throw an exception. + def test_require_decrypt_rejects_non_committing_suite(self): + """REQUIRE_ENCRYPT_REQUIRE_DECRYPT MUST reject non-committing algorithm suites.""" + dec_mats = DecryptionMaterials( + iv=os.urandom(12), + plaintext_data_key=os.urandom(32), + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + ) + pipeline = _make_pipeline( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + keyring_return=dec_mats, + ) + + with pytest.raises(S3EncryptionClientError, match="cannot decrypt non-key-committing"): + pipeline.decrypt(_response(_v2_gcm_metadata())) + + def test_allow_decrypt_accepts_non_committing_suite(self): + """REQUIRE_ENCRYPT_ALLOW_DECRYPT MUST allow non-committing algorithm suites.""" + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + key = os.urandom(32) + iv = os.urandom(12) + plaintext = b"test data for allow-decrypt policy" + ciphertext = AESGCM(key).encrypt(iv, plaintext, None) + + metadata = { + "x-amz-iv": base64.b64encode(iv).decode(), + "x-amz-key-v2": base64.b64encode(b"encrypted-key").decode(), + "x-amz-wrap-alg": "kms+context", + "x-amz-matdesc": "{}", + "x-amz-cek-alg": "AES/GCM/NoPadding", + "x-amz-tag-len": "128", + } + + dec_mats = DecryptionMaterials( + iv=iv, + plaintext_data_key=key, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + ) + pipeline = _make_pipeline( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + keyring_return=dec_mats, + ) + + result = pipeline.decrypt(_response(metadata, ciphertext)) + assert result == plaintext + + +# --------------------------------------------------------------------------- +# Legacy Decryption +# --------------------------------------------------------------------------- + + +class TestLegacyDecryption: + """Tests for specification/s3-encryption/decryption.md#legacy-decryption.""" + + ##= specification/s3-encryption/decryption.md#legacy-decryption + ##= type=test + ##% The S3EC MUST NOT decrypt objects encrypted using legacy unauthenticated algorithm suites + ##% unless specifically configured to do so. + ##= specification/s3-encryption/decryption.md#legacy-decryption + ##= type=test + ##% If the S3EC is not configured to enable legacy unauthenticated content decryption, + ##% the client MUST throw an exception when attempting to decrypt an object encrypted + ##% with a legacy unauthenticated algorithm suite. + def test_legacy_cbc_rejected_by_default(self): + """Legacy CBC objects MUST be rejected unless enable_legacy_unauthenticated_modes is True.""" + dec_mats = DecryptionMaterials( + iv=os.urandom(16), + plaintext_data_key=os.urandom(32), + algorithm_suite=AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF, + ) + pipeline = _make_pipeline( + enable_legacy=False, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + keyring_return=dec_mats, + ) + + with pytest.raises(S3EncryptionClientError, match="not configured to decrypt"): + pipeline.decrypt(_response(_v1_cbc_metadata())) diff --git a/test/test_default_algorithm_commitment.py b/test/test_default_algorithm_commitment.py new file mode 100644 index 00000000..1640451a --- /dev/null +++ b/test/test_default_algorithm_commitment.py @@ -0,0 +1,95 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Integration test: the default encryption algorithm MUST use key commitment. + +When S3EncryptionClientConfig is created with no explicit encryption_algorithm, +the default (ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) MUST produce ciphertext +that is decryptable under REQUIRE_ENCRYPT_REQUIRE_DECRYPT commitment policy. +""" + +import os +from io import BytesIO +from unittest.mock import MagicMock + +from s3_encryption import S3EncryptionClientConfig +from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager +from s3_encryption.materials.encrypted_data_key import EncryptedDataKey +from s3_encryption.materials.keyring import S3Keyring +from s3_encryption.materials.materials import ( + AlgorithmSuite, + CommitmentPolicy, +) +from s3_encryption.pipelines import GetEncryptedObjectPipeline, PutEncryptedObjectPipeline + + +def _mock_keyring(key=None): + """Return a mock keyring that populates encryption/decryption materials.""" + if key is None: + key = os.urandom(32) + mock = MagicMock(spec=S3Keyring) + + def on_encrypt(mats): + mats.plaintext_data_key = key + mats.encrypted_data_key = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms+context", + encrypted_data_key=b"encrypted-key", + ) + return mats + + def on_decrypt(mats, encrypted_data_keys=None): + mats.plaintext_data_key = key + return mats + + mock.on_encrypt.side_effect = on_encrypt + mock.on_decrypt.side_effect = on_decrypt + return mock, key + + +class TestDefaultAlgorithmUsesKeyCommitment: + """The default encryption algorithm MUST be key-committing.""" + + def test_default_config_encrypts_with_committing_algorithm(self): + """S3EncryptionClientConfig with no explicit algorithm MUST default to a + key-committing suite. + """ + keyring, _ = _mock_keyring() + config = S3EncryptionClientConfig(keyring=keyring) + assert config.encryption_algorithm == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + + def test_encryption_materials_defaults_to_committing_algorithm(self): + """EncryptionMaterials with no explicit algorithm MUST default to a + key-committing suite. + """ + from s3_encryption.materials.materials import EncryptionMaterials + + mats = EncryptionMaterials() + assert mats.encryption_algorithm == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + + def test_default_encryption_decryptable_with_require_decrypt(self): + """Ciphertext produced with the default algorithm MUST be decryptable + when the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT. + """ + keyring, key = _mock_keyring() + config = S3EncryptionClientConfig(keyring=keyring) + cmm = DefaultCryptoMaterialsManager(keyring) + + # Encrypt using the default algorithm (no override) + pipeline = PutEncryptedObjectPipeline(cmm, config.encryption_algorithm) + plaintext = b"integration test: default algorithm uses key commitment" + ciphertext, metadata = pipeline.encrypt(plaintext) + + # Build a response dict as if we fetched this object from S3 + response = { + "Body": BytesIO(ciphertext), + "Metadata": metadata, + } + + # Decrypt with REQUIRE_ENCRYPT_REQUIRE_DECRYPT — this will reject + # non-committing algorithm suites, so success proves the default commits. + decrypt_pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + result = decrypt_pipeline.decrypt(response) + assert result == plaintext diff --git a/test/test_encryption.py b/test/test_encryption.py new file mode 100644 index 00000000..a384afbd --- /dev/null +++ b/test/test_encryption.py @@ -0,0 +1,240 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for encryption specification compliance annotations. + +Each test in this module corresponds to a MUST/SHOULD requirement from +specification/s3-encryption/encryption.md and carries a type=test annotation +that mirrors the type=implementation annotation in the source code. +""" + +import base64 +import os +from unittest.mock import MagicMock + +import pytest +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +from s3_encryption.key_derivation import derive_keys +from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager +from s3_encryption.materials.encrypted_data_key import EncryptedDataKey +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy + +_KC_SUITE = AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY +KC_GCM_IV = _KC_SUITE.kc_gcm_iv +MESSAGE_ID_LENGTH = _KC_SUITE.commitment_nonce_length_bytes +SUITE_ID_BYTES = _KC_SUITE.suite_id_bytes +from s3_encryption import S3EncryptionClientConfig +from s3_encryption.pipelines import PutEncryptedObjectPipeline + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _mock_cmm(plaintext_key=None, encrypted_key=b"encrypted-key"): + """Return a CMM backed by a mock keyring that returns the given keys.""" + if plaintext_key is None: + plaintext_key = os.urandom(32) + + mock_keyring = MagicMock() + mock_keyring.on_encrypt.side_effect = lambda mats: _fill_materials( + mats, plaintext_key, encrypted_key + ) + return DefaultCryptoMaterialsManager(mock_keyring), plaintext_key + + +def _fill_materials(mats, plaintext_key, encrypted_key): + mats.plaintext_data_key = plaintext_key + mats.encrypted_data_key = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms+context", + encrypted_data_key=encrypted_key, + ) + return mats + + +# --------------------------------------------------------------------------- +# Content Encryption — General +# --------------------------------------------------------------------------- + + +class TestContentEncryption: + """Tests for specification/s3-encryption/encryption.md#content-encryption.""" + + def test_uses_configured_algorithm_suite(self): + """The pipeline MUST encrypt using the algorithm suite configured in the client.""" + cmm, key = _mock_cmm() + plaintext = b"test data" + + # V2 (GCM no KDF) + config_v2 = S3EncryptionClientConfig( + keyring=MagicMock(), + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + cmm=cmm, + ) + pipeline_v2 = PutEncryptedObjectPipeline(config_v2.cmm, config_v2.encryption_algorithm) + _, meta_v2 = pipeline_v2.encrypt(plaintext) + assert "x-amz-cek-alg" in meta_v2 + assert meta_v2["x-amz-cek-alg"] == "AES/GCM/NoPadding" + + # V3 (KC GCM) + config_v3 = S3EncryptionClientConfig( + keyring=MagicMock(), + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + cmm=cmm, + ) + pipeline_v3 = PutEncryptedObjectPipeline(config_v3.cmm, config_v3.encryption_algorithm) + _, meta_v3 = pipeline_v3.encrypt(plaintext) + assert "x-amz-c" in meta_v3 + assert meta_v3["x-amz-c"] == "115" + + ##= specification/s3-encryption/encryption.md#content-encryption + ##= type=test + ##% The client MUST generate an IV or Message ID using the length of the IV + ##% or Message ID defined in the algorithm suite. + def test_iv_generated_with_correct_length_gcm(self): + """GCM encryption MUST produce a 12-byte IV.""" + cmm, _ = _mock_cmm() + pipeline = PutEncryptedObjectPipeline(cmm, AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + + _, meta = pipeline.encrypt(b"test") + iv_bytes = base64.b64decode(meta["x-amz-iv"]) + assert len(iv_bytes) == 12 + + def test_message_id_generated_with_correct_length_kc(self): + """KC-GCM encryption MUST produce a 28-byte Message ID.""" + cmm, _ = _mock_cmm() + pipeline = PutEncryptedObjectPipeline( + cmm, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) + + _, meta = pipeline.encrypt(b"test") + message_id_bytes = base64.b64decode(meta["x-amz-i"]) + assert len(message_id_bytes) == MESSAGE_ID_LENGTH + + ##= specification/s3-encryption/encryption.md#content-encryption + ##= type=test + ##% The generated IV or Message ID MUST be set or returned from the encryption + ##% process such that it can be included in the content metadata. + def test_iv_included_in_metadata_gcm(self): + """GCM encryption MUST include the IV in the returned metadata.""" + cmm, _ = _mock_cmm() + pipeline = PutEncryptedObjectPipeline(cmm, AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + + _, meta = pipeline.encrypt(b"test") + assert "x-amz-iv" in meta + + def test_message_id_included_in_metadata_kc(self): + """KC-GCM encryption MUST include the Message ID in the returned metadata.""" + cmm, _ = _mock_cmm() + pipeline = PutEncryptedObjectPipeline( + cmm, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) + + _, meta = pipeline.encrypt(b"test") + assert "x-amz-i" in meta + + +# --------------------------------------------------------------------------- +# ALG_AES_256_GCM_IV12_TAG16_NO_KDF +# --------------------------------------------------------------------------- + + +class TestGcmNoKdf: + """Tests for specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf.""" + + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf + ##= type=test + ##% The client MUST initialize the cipher, or call an AES-GCM encryption API, + ##% with the plaintext data key, the generated IV, and the tag length defined + ##% in the Algorithm Suite when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf + ##= type=test + ##% The client MUST NOT provide any AAD when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + def test_gcm_encrypt_decrypt_roundtrip_no_aad(self): + """GCM encryption MUST use the data key, generated IV, and no AAD.""" + cmm, key = _mock_cmm() + pipeline = PutEncryptedObjectPipeline(cmm, AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + plaintext = b"roundtrip test for GCM no KDF" + + ciphertext, meta = pipeline.encrypt(plaintext) + + # Decrypt with the same key, IV, and no AAD + iv = base64.b64decode(meta["x-amz-iv"]) + aesgcm = AESGCM(key) + decrypted = aesgcm.decrypt(nonce=iv, data=ciphertext, associated_data=None) + assert decrypted == plaintext + + def test_gcm_decrypt_fails_with_aad(self): + """Ciphertext produced with no AAD MUST NOT decrypt with AAD.""" + cmm, key = _mock_cmm() + pipeline = PutEncryptedObjectPipeline(cmm, AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + + ciphertext, meta = pipeline.encrypt(b"test") + + iv = base64.b64decode(meta["x-amz-iv"]) + aesgcm = AESGCM(key) + with pytest.raises(Exception): + aesgcm.decrypt(nonce=iv, data=ciphertext, associated_data=b"unexpected-aad") + + +# --------------------------------------------------------------------------- +# ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY +# --------------------------------------------------------------------------- + + +class TestKcGcm: + """Tests for specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key.""" + + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key + ##= type=test + ##% The client MUST use HKDF to derive the key commitment value and the derived + ##% encrypting key as described in [Key Derivation](key-derivation.md). + def test_kc_gcm_uses_hkdf_derived_key(self): + """KC-GCM encryption MUST use HKDF-derived keys, not the raw data key.""" + cmm, raw_key = _mock_cmm() + pipeline = PutEncryptedObjectPipeline( + cmm, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) + plaintext = b"roundtrip test for KC GCM" + + ciphertext, meta = pipeline.encrypt(plaintext) + + message_id = base64.b64decode(meta["x-amz-i"]) + derived_key, _ = derive_keys( + raw_key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) + + # Decrypt with the HKDF-derived key, fixed IV, and suite ID as AAD + aesgcm = AESGCM(derived_key) + decrypted = aesgcm.decrypt(nonce=KC_GCM_IV, data=ciphertext, associated_data=SUITE_ID_BYTES) + assert decrypted == plaintext + + # Decrypting with the raw key must fail + aesgcm_raw = AESGCM(raw_key) + with pytest.raises(Exception): + aesgcm_raw.decrypt(nonce=KC_GCM_IV, data=ciphertext, associated_data=SUITE_ID_BYTES) + + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key + ##= type=test + ##% The derived key commitment value MUST be set or returned from the encryption + ##% process such that it can be included in the content metadata. + def test_kc_gcm_commitment_in_metadata(self): + """KC-GCM encryption MUST include the key commitment in metadata.""" + cmm, raw_key = _mock_cmm() + pipeline = PutEncryptedObjectPipeline( + cmm, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) + + _, meta = pipeline.encrypt(b"test") + + assert "x-amz-d" in meta + commitment_bytes = base64.b64decode(meta["x-amz-d"]) + + # Verify the commitment matches what HKDF would produce + message_id = base64.b64decode(meta["x-amz-i"]) + _, expected_commitment = derive_keys( + raw_key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) + assert commitment_bytes == expected_commitment diff --git a/test/test_encryption_materials_integration.py b/test/test_encryption_materials_integration.py index 4b6bfefd..e9e59023 100644 --- a/test/test_encryption_materials_integration.py +++ b/test/test_encryption_materials_integration.py @@ -35,7 +35,7 @@ def test_keyring_on_encrypt(self): assert isinstance(result, EncryptionMaterials) assert result.encryption_context == { "key1": "value1", - "aws:x-amz-cek-alg": "AES/GCM/NoPadding", + "aws:x-amz-cek-alg": "115", } def test_cmm_get_encryption_materials_with_dict(self): diff --git a/test/test_key_commitment.py b/test/test_key_commitment.py new file mode 100644 index 00000000..a82fc9fd --- /dev/null +++ b/test/test_key_commitment.py @@ -0,0 +1,157 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for key commitment policy specification compliance annotations. + +Each test in this module corresponds to a MUST/SHOULD requirement from +specification/s3-encryption/key-commitment.md and carries a type=test annotation +that mirrors the type=implementation annotation in the source code. +""" + +import base64 +import os +from io import BytesIO +from unittest.mock import Mock + +import pytest +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.key_derivation import derive_keys +from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager +from s3_encryption.materials.keyring import S3Keyring +from s3_encryption.materials.materials import ( + AlgorithmSuite, + CommitmentPolicy, + DecryptionMaterials, +) +from s3_encryption.pipelines import GetEncryptedObjectPipeline + +_KC_SUITE = AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY +KC_GCM_IV = _KC_SUITE.kc_gcm_iv +SUITE_ID_BYTES = _KC_SUITE.suite_id_bytes + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_pipeline(commitment_policy, keyring_return=None): + """Create a GetEncryptedObjectPipeline with a mocked CMM/keyring.""" + mock_keyring = Mock(spec=S3Keyring) + if keyring_return is not None: + mock_keyring.on_decrypt.return_value = keyring_return + cmm = DefaultCryptoMaterialsManager(mock_keyring) + return GetEncryptedObjectPipeline( + cmm, + commitment_policy=commitment_policy, + enable_legacy_unauthenticated_modes=True, + ) + + +def _v2_gcm_response(key, plaintext=b"test data"): + """Create a V2 GCM-encrypted response with real ciphertext.""" + iv = os.urandom(12) + ciphertext = AESGCM(key).encrypt(iv, plaintext, None) + metadata = { + "x-amz-iv": base64.b64encode(iv).decode(), + "x-amz-key-v2": base64.b64encode(b"encrypted-key").decode(), + "x-amz-wrap-alg": "kms+context", + "x-amz-matdesc": "{}", + "x-amz-cek-alg": "AES/GCM/NoPadding", + "x-amz-tag-len": "128", + } + dec_mats = DecryptionMaterials( + iv=iv, + plaintext_data_key=key, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + ) + return {"Body": BytesIO(ciphertext), "Metadata": metadata}, dec_mats, plaintext + + +def _v3_kc_gcm_response(key, plaintext=b"test data"): + """Create a V3 KC-GCM-encrypted response with real ciphertext.""" + message_id = os.urandom(28) + derived_key, commitment = derive_keys( + key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) + ciphertext = AESGCM(derived_key).encrypt(KC_GCM_IV, plaintext, SUITE_ID_BYTES) + metadata = { + "x-amz-c": "115", + "x-amz-3": base64.b64encode(b"encrypted-key").decode(), + "x-amz-w": "12", + "x-amz-t": "{}", + "x-amz-d": base64.b64encode(commitment).decode(), + "x-amz-i": base64.b64encode(message_id).decode(), + } + dec_mats = DecryptionMaterials( + plaintext_data_key=key, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ) + return {"Body": BytesIO(ciphertext), "Metadata": metadata}, dec_mats, plaintext + + +# --------------------------------------------------------------------------- +# Commitment Policy Tests +# --------------------------------------------------------------------------- + + +class TestCommitmentPolicy: + """Tests for specification/s3-encryption/key-commitment.md#commitment-policy.""" + + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##= type=test + ##% When the commitment policy is FORBID_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST allow decryption using algorithm suites which do not support key commitment. + def test_forbid_encrypt_allows_non_committing_decrypt(self): + """FORBID_ENCRYPT_ALLOW_DECRYPT MUST allow decryption with non-committing suites.""" + key = os.urandom(32) + response, dec_mats, plaintext = _v2_gcm_response(key) + + pipeline = _make_pipeline( + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + keyring_return=dec_mats, + ) + result = pipeline.decrypt(response) + assert result == plaintext + + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##= type=test + ##% When the commitment policy is REQUIRE_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST allow decryption using algorithm suites which do not support key commitment. + def test_require_encrypt_allow_decrypt_allows_non_committing_decrypt(self): + """REQUIRE_ENCRYPT_ALLOW_DECRYPT MUST allow decryption with non-committing suites.""" + key = os.urandom(32) + response, dec_mats, plaintext = _v2_gcm_response(key) + + pipeline = _make_pipeline( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + keyring_return=dec_mats, + ) + result = pipeline.decrypt(response) + assert result == plaintext + + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##= type=test + ##% When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, the S3EC MUST NOT allow decryption using algorithm suites which do not support key commitment. + def test_require_require_rejects_non_committing_decrypt(self): + """REQUIRE_ENCRYPT_REQUIRE_DECRYPT MUST reject non-committing algorithm suites.""" + key = os.urandom(32) + response, dec_mats, _ = _v2_gcm_response(key) + + pipeline = _make_pipeline( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + keyring_return=dec_mats, + ) + with pytest.raises(S3EncryptionClientError, match="cannot decrypt non-key-committing"): + pipeline.decrypt(response) + + def test_require_require_allows_committing_decrypt(self): + """REQUIRE_ENCRYPT_REQUIRE_DECRYPT MUST allow decryption with committing suites.""" + key = os.urandom(32) + response, dec_mats, plaintext = _v3_kc_gcm_response(key) + + pipeline = _make_pipeline( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + keyring_return=dec_mats, + ) + result = pipeline.decrypt(response) + assert result == plaintext diff --git a/test/test_key_commitment_encrypt.py b/test/test_key_commitment_encrypt.py new file mode 100644 index 00000000..ed8e8f3e --- /dev/null +++ b/test/test_key_commitment_encrypt.py @@ -0,0 +1,108 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for key commitment policy enforcement on the encryption path. + +Per specification/s3-encryption/key-commitment.md#commitment-policy: + - REQUIRE_ENCRYPT_ALLOW_DECRYPT: the S3EC MUST only encrypt using an + algorithm suite which supports key commitment. + - REQUIRE_ENCRYPT_REQUIRE_DECRYPT: the S3EC MUST only encrypt using an + algorithm suite which supports key commitment. + - FORBID_ENCRYPT_ALLOW_DECRYPT: the S3EC MUST NOT encrypt using an + algorithm suite which supports key commitment. + +Per specification/s3-encryption/client.md#key-commitment: + - The S3EC MUST validate the configured Encryption Algorithm against the + provided key commitment policy. + - If the configured Encryption Algorithm is incompatible with the key + commitment policy, then it MUST throw an exception. + +These tests verify that the S3EC rejects mismatched commitment policy and +algorithm suite configurations. The rejection may occur at client config +creation time or at encrypt time. +""" + +import os +from unittest.mock import MagicMock + +import pytest + +from s3_encryption import S3EncryptionClientConfig +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.materials.encrypted_data_key import EncryptedDataKey +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _mock_keyring(): + """Return a mock keyring that populates encryption materials.""" + key = os.urandom(32) + mock = MagicMock() + + def on_encrypt(mats): + mats.plaintext_data_key = key + mats.encrypted_data_key = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms+context", + encrypted_data_key=b"encrypted-key", + ) + return mats + + mock.on_encrypt.side_effect = on_encrypt + return mock + + +# --------------------------------------------------------------------------- +# REQUIRE_ENCRYPT_* with non-committing algorithm → MUST fail +# --------------------------------------------------------------------------- + + +class TestRequireEncryptRejectsNonCommitting: + """Configuring REQUIRE_ENCRYPT_* with a non-committing algorithm MUST fail.""" + + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##= type=test + ##% When the commitment policy is REQUIRE_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST only encrypt using an algorithm suite which supports key commitment. + def test_require_encrypt_allow_decrypt_rejects_non_committing_gcm(self): + keyring = _mock_keyring() + with pytest.raises(S3EncryptionClientError): + S3EncryptionClientConfig( + keyring=keyring, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + ) + + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##= type=test + ##% When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, the S3EC MUST only encrypt using an algorithm suite which supports key commitment. + def test_require_encrypt_require_decrypt_rejects_non_committing_gcm(self): + keyring = _mock_keyring() + with pytest.raises(S3EncryptionClientError): + S3EncryptionClientConfig( + keyring=keyring, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + + +# --------------------------------------------------------------------------- +# FORBID_ENCRYPT_ALLOW_DECRYPT with committing algorithm → MUST fail +# --------------------------------------------------------------------------- + + +class TestForbidEncryptRejectsCommitting: + """Configuring FORBID_ENCRYPT_ALLOW_DECRYPT with a committing algorithm MUST fail.""" + + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##= type=test + ##% When the commitment policy is FORBID_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST NOT encrypt using an algorithm suite which supports key commitment. + def test_forbid_encrypt_allow_decrypt_rejects_committing_gcm(self): + keyring = _mock_keyring() + with pytest.raises(S3EncryptionClientError): + S3EncryptionClientConfig( + keyring=keyring, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) diff --git a/test/test_key_derivation.py b/test/test_key_derivation.py new file mode 100644 index 00000000..0bec87f7 --- /dev/null +++ b/test/test_key_derivation.py @@ -0,0 +1,281 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for key derivation specification compliance annotations. + +Each test in this module corresponds to a MUST requirement from +specification/s3-encryption/key-derivation.md and carries a type=test annotation +that mirrors the type=implementation annotation in the source code. +""" + +import hmac +import os + +import pytest +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.hashes import SHA512 +from cryptography.hazmat.primitives.kdf.hkdf import HKDFExpand + +from s3_encryption.key_derivation import ( + derive_keys, +) +from s3_encryption.materials.materials import AlgorithmSuite + +_KC_SUITE = AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY +SUITE_ID_BYTES = _KC_SUITE.suite_id_bytes +ENCRYPTION_KEY_LENGTH = _KC_SUITE.data_key_length_bytes +COMMIT_KEY_LENGTH = _KC_SUITE.commitment_length_bytes +MESSAGE_ID_LENGTH = _KC_SUITE.commitment_nonce_length_bytes +KC_GCM_IV = _KC_SUITE.kc_gcm_iv + + +# --------------------------------------------------------------------------- +# HKDF Extract / Expand +# --------------------------------------------------------------------------- + + +class TestHkdfOperation: + """Tests for specification/s3-encryption/key-derivation.md#hkdf-operation.""" + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The hash function MUST be specified by the algorithm suite commitment settings. + def test_hash_function_is_sha512(self): + """HKDF extract MUST use the hash function specified by the algorithm suite.""" + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) + msg_id = os.urandom(MESSAGE_ID_LENGTH) + + # Manual extract using the algorithm suite's configured hash + hash_alg = _KC_SUITE.kdf_hash_algorithm + prk = hmac.new(msg_id, pdk, hash_alg).digest() + + # Expand with the same hash to get expected DEK + expected_dek = HKDFExpand( + algorithm=SHA512(), + length=_KC_SUITE.data_key_length_bytes, + info=SUITE_ID_BYTES + b"DERIVEKEY", + ).derive(prk) + + # derive_keys using the suite must match + actual_dek, _ = derive_keys(pdk, msg_id, _KC_SUITE) + assert actual_dek == expected_dek + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The input keying material MUST be the plaintext data key (PDK) generated by the key provider. + def test_ikm_is_plaintext_data_key(self): + """Different plaintext data keys MUST produce different derived keys.""" + msg_id = os.urandom(MESSAGE_ID_LENGTH) + pdk_a = os.urandom(_KC_SUITE.data_key_length_bytes) + pdk_b = os.urandom(_KC_SUITE.data_key_length_bytes) + + key_a, _ = derive_keys(pdk_a, msg_id, _KC_SUITE) + key_b, _ = derive_keys(pdk_b, msg_id, _KC_SUITE) + assert key_a != key_b + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The length of the input keying material MUST equal the key derivation input length specified by the algorithm suite commit key derivation setting. + def test_ikm_length_is_32_bytes(self): + """The plaintext data key (IKM) length MUST equal the algorithm suite's data key length.""" + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) + msg_id = os.urandom(MESSAGE_ID_LENGTH) + assert len(pdk) == _KC_SUITE.data_key_length_bytes + # Should succeed with correct-length key + key, ck = derive_keys(pdk, msg_id, _KC_SUITE) + assert len(key) == ENCRYPTION_KEY_LENGTH + assert len(ck) == COMMIT_KEY_LENGTH + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The length of the input keying material MUST equal the key derivation input length specified by the algorithm suite commit key derivation setting. + def test_ikm_wrong_length_raises(self): + """derive_keys MUST raise when the plaintext data key length doesn't match the suite.""" + from s3_encryption.exceptions import S3EncryptionClientError + + msg_id = os.urandom(MESSAGE_ID_LENGTH) + # Too short + with pytest.raises(S3EncryptionClientError): + derive_keys(os.urandom(16), msg_id, _KC_SUITE) + # Too long + with pytest.raises(S3EncryptionClientError): + derive_keys(os.urandom(64), msg_id, _KC_SUITE) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The salt MUST be the Message ID with the length defined in the algorithm suite. + def test_salt_is_message_id(self): + """Different Message IDs (salts) MUST produce different derived keys.""" + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) + msg_id_a = os.urandom(MESSAGE_ID_LENGTH) + msg_id_b = os.urandom(MESSAGE_ID_LENGTH) + + key_a, _ = derive_keys(pdk, msg_id_a, _KC_SUITE) + key_b, _ = derive_keys(pdk, msg_id_b, _KC_SUITE) + assert key_a != key_b + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The DEK input pseudorandom key MUST be the output from the extract step. + def test_dek_uses_prk_from_extract(self): + """The DEK expand step MUST use the PRK from the extract step.""" + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) + msg_id = os.urandom(MESSAGE_ID_LENGTH) + + # Manual extract + prk = hmac.new(msg_id, pdk, _KC_SUITE.kdf_hash_algorithm).digest() + # Manual expand for DEK + expected_dek = HKDFExpand( + algorithm=SHA512(), + length=ENCRYPTION_KEY_LENGTH, + info=SUITE_ID_BYTES + b"DERIVEKEY", + ).derive(prk) + + actual_dek, _ = derive_keys(pdk, msg_id, _KC_SUITE) + assert actual_dek == expected_dek + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The length of the output keying material MUST equal the encryption key length specified by the algorithm suite encryption settings. + def test_dek_output_length(self): + """The derived encryption key MUST match the encryption key length from the algorithm suite.""" + key, _ = derive_keys( + os.urandom(_KC_SUITE.data_key_length_bytes), os.urandom(MESSAGE_ID_LENGTH), _KC_SUITE + ) + assert len(key) == _KC_SUITE.data_key_length_bytes + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The input info MUST be a concatenation of the algorithm suite ID as bytes followed by the string DERIVEKEY as UTF8 encoded bytes. + def test_dek_info_is_suite_id_plus_derivekey(self): + """DEK expand info MUST be suite_id_bytes + b'DERIVEKEY'.""" + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) + msg_id = os.urandom(MESSAGE_ID_LENGTH) + + prk = hmac.new(msg_id, pdk, _KC_SUITE.kdf_hash_algorithm).digest() + + # Correct info + correct_dek = HKDFExpand( + algorithm=SHA512(), + length=ENCRYPTION_KEY_LENGTH, + info=SUITE_ID_BYTES + b"DERIVEKEY", + ).derive(prk) + + # Wrong info should produce different output + wrong_dek = HKDFExpand( + algorithm=SHA512(), + length=ENCRYPTION_KEY_LENGTH, + info=SUITE_ID_BYTES + b"WRONGKEY", + ).derive(prk) + + actual_dek, _ = derive_keys(pdk, msg_id, _KC_SUITE) + assert actual_dek == correct_dek + assert actual_dek != wrong_dek + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The CK input pseudorandom key MUST be the output from the extract step. + def test_ck_uses_prk_from_extract(self): + """The CK expand step MUST use the PRK from the extract step.""" + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) + msg_id = os.urandom(MESSAGE_ID_LENGTH) + + prk = hmac.new(msg_id, pdk, _KC_SUITE.kdf_hash_algorithm).digest() + expected_ck = HKDFExpand( + algorithm=SHA512(), + length=COMMIT_KEY_LENGTH, + info=SUITE_ID_BYTES + b"COMMITKEY", + ).derive(prk) + + _, actual_ck = derive_keys(pdk, msg_id, _KC_SUITE) + assert actual_ck == expected_ck + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The length of the output keying material MUST equal the commit key length specified by the supported algorithm suites. + def test_ck_output_length(self): + """The commit key length MUST match the algorithm suite's commitment length.""" + _, ck = derive_keys( + os.urandom(_KC_SUITE.data_key_length_bytes), os.urandom(MESSAGE_ID_LENGTH), _KC_SUITE + ) + assert len(ck) == _KC_SUITE.commitment_length_bytes + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The input info MUST be a concatenation of the algorithm suite ID as bytes followed by the string COMMITKEY as UTF8 encoded bytes. + def test_ck_info_is_suite_id_plus_commitkey(self): + """CK expand info MUST be suite_id_bytes + b'COMMITKEY'.""" + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) + msg_id = os.urandom(MESSAGE_ID_LENGTH) + + prk = hmac.new(msg_id, pdk, _KC_SUITE.kdf_hash_algorithm).digest() + + correct_ck = HKDFExpand( + algorithm=SHA512(), + length=COMMIT_KEY_LENGTH, + info=SUITE_ID_BYTES + b"COMMITKEY", + ).derive(prk) + + wrong_ck = HKDFExpand( + algorithm=SHA512(), + length=COMMIT_KEY_LENGTH, + info=SUITE_ID_BYTES + b"WRONGKEY", + ).derive(prk) + + _, actual_ck = derive_keys(pdk, msg_id, _KC_SUITE) + assert actual_ck == correct_ck + assert actual_ck != wrong_ck + + +# --------------------------------------------------------------------------- +# IV and AAD for KC-GCM +# --------------------------------------------------------------------------- + + +class TestKcGcmCipherParams: + """Tests for KC-GCM cipher initialization parameters.""" + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% When encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ##% the IV used in the AES-GCM content encryption/decryption MUST consist entirely of bytes with the value 0x01. + def test_kc_gcm_iv_is_all_0x01(self): + """The KC-GCM IV MUST consist entirely of 0x01 bytes.""" + assert all(b == 0x01 for b in KC_GCM_IV) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% The IV's total length MUST match the IV length defined by the algorithm suite. + def test_kc_gcm_iv_length_is_12(self): + """The KC-GCM IV length MUST match the IV length defined by the algorithm suite.""" + assert len(KC_GCM_IV) == _KC_SUITE.cipher_iv_length_bytes + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% The client MUST initialize the cipher, or call an AES-GCM encryption API, with the derived encryption key, an IV containing only bytes with the value 0x01, + ##% and the tag length defined in the Algorithm Suite when encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. + def test_kc_gcm_roundtrip_with_derived_key_iv_aad(self): + """KC-GCM MUST encrypt/decrypt with derived key, 0x01 IV, and suite ID as AAD.""" + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) + msg_id = os.urandom(MESSAGE_ID_LENGTH) + plaintext = b"key derivation roundtrip test" + + derived_key, _ = derive_keys(pdk, msg_id, _KC_SUITE) + + # Encrypt with derived key, KC_GCM_IV, and SUITE_ID_BYTES as AAD + aesgcm = AESGCM(derived_key) + ciphertext = aesgcm.encrypt(KC_GCM_IV, plaintext, SUITE_ID_BYTES) + + # Decrypt with same parameters + decrypted = aesgcm.decrypt(KC_GCM_IV, ciphertext, SUITE_ID_BYTES) + assert decrypted == plaintext + + # Decrypting with wrong AAD must fail + with pytest.raises(Exception): + aesgcm.decrypt(KC_GCM_IV, ciphertext, b"\x00\x00") + + # Decrypting with wrong IV must fail + with pytest.raises(Exception): + aesgcm.decrypt(b"\x00" * _KC_SUITE.cipher_iv_length_bytes, ciphertext, SUITE_ID_BYTES) diff --git a/test/test_kms_keyring.py b/test/test_kms_keyring.py index d613cbf9..0a5d66de 100644 --- a/test/test_kms_keyring.py +++ b/test/test_kms_keyring.py @@ -128,7 +128,7 @@ def test_on_encrypt_adds_kms_context_algorithm(self): result = keyring.on_encrypt(enc_materials) call_args = mock_kms_client.generate_data_key.call_args - assert call_args.kwargs["EncryptionContext"]["aws:x-amz-cek-alg"] == "AES/GCM/NoPadding" + assert call_args.kwargs["EncryptionContext"]["aws:x-amz-cek-alg"] == "115" def test_on_encrypt_sets_encrypted_data_key(self): """Test that on_encrypt sets the encrypted data key from KMS response.""" diff --git a/test/test_metadata.py b/test/test_metadata.py index ba783bf5..55c1f0b2 100644 --- a/test/test_metadata.py +++ b/test/test_metadata.py @@ -56,7 +56,7 @@ def test_to_dict(self): # Verify that fields that are None are not included in the dictionary assert "x-amz-key" not in metadata_dict assert "x-amz-matdesc" not in metadata_dict - # Note: content_cipher_tag_length has a default value of "128" + # content_cipher_tag_length defaults to "128" for V1/V2 assert metadata_dict.get("x-amz-tag-len") == "128" assert "x-amz-crypto-instr-file" not in metadata_dict @@ -124,6 +124,9 @@ def test_to_dict_v3_fields(self): assert metadata_dict["x-amz-m"] == "mat-desc" assert metadata_dict["x-amz-t"] == "encryption-context" + # V3 metadata must NOT include V1/V2-only keys like x-amz-tag-len + assert "x-amz-tag-len" not in metadata_dict + def test_is_v1_format(self): metadata = ObjectMetadata( content_iv="iv", diff --git a/test/test_pipelines.py b/test/test_pipelines.py index 9f40cd5c..3d32b8cc 100644 --- a/test/test_pipelines.py +++ b/test/test_pipelines.py @@ -8,8 +8,10 @@ import pytest +from s3_encryption.exceptions import S3EncryptionClientError, S3EncryptionClientSecurityError from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager from s3_encryption.materials.keyring import S3Keyring +from s3_encryption.materials.materials import CommitmentPolicy, DecryptionMaterials from s3_encryption.pipelines import GetEncryptedObjectPipeline @@ -45,7 +47,11 @@ def test_decrypt_v1_from_instruction_file(self): cmm = DefaultCryptoMaterialsManager(mock_keyring) # Create pipeline with mocked S3 client - pipeline = GetEncryptedObjectPipeline(cmm, mock_s3_client) + pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + s3_client=mock_s3_client, + ) # Create mock response mock_response = { @@ -100,7 +106,11 @@ def test_decrypt_v2_from_instruction_file(self): cmm = DefaultCryptoMaterialsManager(mock_keyring) # Create pipeline with mocked S3 client - pipeline = GetEncryptedObjectPipeline(cmm, mock_s3_client) + pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + s3_client=mock_s3_client, + ) # Create mock response mock_response = { @@ -127,7 +137,7 @@ def test_decrypt_v2_from_instruction_file(self): ##% In the V3 message format, only the content metadata related to ##% the encrypted data is stored in the Instruction File. def test_decrypt_v3_from_instruction_file(self): - """Test decrypting V3 format with instruction file.""" + """Test decrypting V3 format with instruction file (kms+context wrapping).""" # Object metadata contains V3 content keys only object_metadata = { "x-amz-c": "115", # Compressed algorithm suite @@ -136,11 +146,11 @@ def test_decrypt_v3_from_instruction_file(self): } # Instruction file contains encrypted data key and wrapping algorithm + # Uses "12" (kms+context) with "x-amz-t" for encryption context instruction_file_metadata = { "x-amz-3": base64.b64encode(b"encrypted-key-data").decode("utf-8"), - "x-amz-w": "02", # AES/GCM - "x-amz-m": json.dumps({"test-instruction": "material-desc-instruction"}), - "x-amz-crypto-instr-file": "", + "x-amz-w": "12", # kms+context + "x-amz-t": json.dumps({"aws:x-amz-cek-alg": "AES/GCM/NoPadding"}), } # Create mock S3 client @@ -156,7 +166,11 @@ def test_decrypt_v3_from_instruction_file(self): cmm = DefaultCryptoMaterialsManager(mock_keyring) # Create pipeline with mocked S3 client - pipeline = GetEncryptedObjectPipeline(cmm, mock_s3_client) + pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + s3_client=mock_s3_client, + ) # Create mock response with encrypted data iv = os.urandom(12) @@ -168,8 +182,6 @@ def test_decrypt_v3_from_instruction_file(self): } # Mock the keyring to return decryption materials - from s3_encryption.materials.materials import DecryptionMaterials - plaintext_data_key = os.urandom(32) mock_dec_materials = DecryptionMaterials( @@ -182,8 +194,11 @@ def test_decrypt_v3_from_instruction_file(self): mock_keyring.on_decrypt.return_value = mock_dec_materials - # This should fail with NotImplementedError since V3 decryption isn't implemented yet - with pytest.raises(NotImplementedError, match="V3 decryption not yet implemented"): + # V3 decryption is now implemented; with fake commitment data, + # key commitment verification will fail. + with pytest.raises( + S3EncryptionClientSecurityError, match="Key commitment verification failed" + ): pipeline.decrypt(mock_response, bucket="test-bucket", key="test-key") # Verify instruction file was fetched @@ -217,7 +232,11 @@ def test_decrypt_with_custom_instruction_file_suffix(self): mock_keyring = Mock(spec=S3Keyring) cmm = DefaultCryptoMaterialsManager(mock_keyring) - pipeline = GetEncryptedObjectPipeline(cmm, mock_s3_client) + pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + s3_client=mock_s3_client, + ) mock_response = { "Body": BytesIO(b"encrypted-test-data"), @@ -239,3 +258,36 @@ def test_decrypt_with_custom_instruction_file_suffix(self): mock_s3_client.get_object.assert_called_once_with( Bucket="test-bucket", Key="test-key.custom-suffix" ) + + def test_decrypt_v3_unsupported_wrap_alg(self): + """Test that V3 decryption with unsupported wrapping algorithm is rejected by the keyring.""" + # V3 metadata with AES/GCM wrapping (02) — not supported by the KMS keyring + metadata = { + "x-amz-c": "115", + "x-amz-3": base64.b64encode(b"encrypted-key-data").decode("utf-8"), + "x-amz-w": "02", # AES/GCM — unsupported by KMS keyring + "x-amz-m": json.dumps({"some": "material-desc"}), + "x-amz-d": base64.b64encode(b"key-commitment-data").decode("utf-8"), + "x-amz-i": base64.b64encode(b"test-message-id").decode("utf-8"), + } + + mock_keyring = Mock(spec=S3Keyring) + # The keyring rejects wrapping algorithms it doesn't support + mock_keyring.on_decrypt.side_effect = S3EncryptionClientError( + "AES/GCM is not a valid key wrapping algorithm!" + ) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + + mock_response = { + "Body": BytesIO(b"encrypted-test-data"), + "Metadata": metadata, + } + + with pytest.raises( + S3EncryptionClientError, match="AES/GCM is not a valid key wrapping algorithm" + ): + pipeline.decrypt(mock_response) From e70cf78aa90e7fd3fcb9784ac546fec183220359 Mon Sep 17 00:00:00 2001 From: Tony Knapp <5892063+texastony@users.noreply.github.com> Date: Fri, 20 Mar 2026 12:13:03 -0700 Subject: [PATCH 60/81] chore: add pytest-cov coverage reporting and update GitHub Actions (#154) * chore: add pytest-cov for line coverage reporting * chore: update actions/checkout to v6, actions/cache to v5, actions/upload-artifact to v7, and actions/setup-python to v6 * chore: update aws-actions/configure-aws-credentials to v6 * ci: add coverage threshold check (fail-under=93%) * ci: suggest incrementing coverage threshold when exceeded --- .github/workflows/duvet-test-server.yml | 10 ++++----- .github/workflows/duvet.yml | 2 +- .github/workflows/lint.yml | 4 ++-- .github/workflows/python-integ.yml | 29 +++++++++++++++++++++---- .github/workflows/test-server.yml | 12 +++++----- .gitignore | 2 ++ Makefile | 10 ++++----- pyproject.toml | 7 ++++++ 8 files changed, 53 insertions(+), 23 deletions(-) diff --git a/.github/workflows/duvet-test-server.yml b/.github/workflows/duvet-test-server.yml index f8f6e2ac..58ae19a2 100644 --- a/.github/workflows/duvet-test-server.yml +++ b/.github/workflows/duvet-test-server.yml @@ -14,11 +14,11 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v6 # There are a lot of submodules here # This initializes the checkouts in parallel (--jobs) - # rather than in series the way actions/checkout@v5 does it. + # rather than in series the way actions/checkout@v6 does it. - name: Get CPU count id: cpu-count @@ -42,7 +42,7 @@ jobs: - name: Checkout CPP code cpp-v3 - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: submodules: recursive repository: aws/aws-sdk-cpp @@ -64,7 +64,7 @@ jobs: - name: Upload duvet reports if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: test-server-reports include-hidden-files: true @@ -95,7 +95,7 @@ jobs: - name: Upload compliance dashboard if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: compliance-dashboard include-hidden-files: true diff --git a/.github/workflows/duvet.yml b/.github/workflows/duvet.yml index eb7b49e2..23bbe45a 100644 --- a/.github/workflows/duvet.yml +++ b/.github/workflows/duvet.yml @@ -32,7 +32,7 @@ jobs: run: make duvet - name: Upload duvet reports - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: reports include-hidden-files: true diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index bb1655bb..b374a9a7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,10 +12,10 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.11' diff --git a/.github/workflows/python-integ.yml b/.github/workflows/python-integ.yml index 9e5ae818..b845e725 100644 --- a/.github/workflows/python-integ.yml +++ b/.github/workflows/python-integ.yml @@ -18,17 +18,17 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: submodules: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ inputs.python-version || '3.11' }} - name: Cache uv dependencies - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.cache/uv key: ${{ runner.os }}-uv-${{ hashFiles('./test-server/python-v3-server/**/pyproject.toml') }} @@ -42,7 +42,7 @@ jobs: run: make install - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@v6 with: role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role aws-region: us-west-2 @@ -55,3 +55,24 @@ jobs: env: CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} + + - name: Generate coverage HTML report + if: always() + run: uv run coverage html -d coverage-report + + - name: Upload coverage report + if: always() + uses: actions/upload-artifact@v7 + with: + name: coverage-report + path: coverage-report/ + + - name: Check coverage threshold + run: | + THRESHOLD=93 + ACTUAL=$(uv run coverage report --format=total) + echo "Coverage: ${ACTUAL}% (threshold: ${THRESHOLD}%)" + if [ "$ACTUAL" -gt "$THRESHOLD" ]; then + echo "::warning::Coverage is ${ACTUAL}%, consider updating --fail-under to ${ACTUAL} in python-integ.yml" + fi + uv run coverage report --fail-under=$THRESHOLD diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml index 80991a99..7b26c3fc 100644 --- a/.github/workflows/test-server.yml +++ b/.github/workflows/test-server.yml @@ -19,14 +19,14 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: submodules: false token: ${{ secrets.PAT_FOR_SPEC }} # There are a lot of submodules here # This initializes the checkouts in parallel (--jobs) - # rather than in series the way actions/checkout@v5 does it. + # rather than in series the way actions/checkout@v6 does it. - name: Get CPU count id: cpu-count @@ -89,7 +89,7 @@ jobs: # Cache Gradle dependencies and build outputs - name: Cache Gradle packages - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~/.gradle/caches @@ -100,7 +100,7 @@ jobs: ${{ runner.os }}-gradle- - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@v6 with: role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role aws-region: us-west-2 @@ -133,7 +133,7 @@ jobs: - name: Upload server logs if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: server-logs path: | @@ -144,7 +144,7 @@ jobs: - name: Upload results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: results path: test-server/java-tests/build/reports/tests/integ diff --git a/.gitignore b/.gitignore index 5cd8f239..3691eef4 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,5 @@ smithy-java-core/out # test server *.pid +.coverage +coverage-report/ diff --git a/Makefile b/Makefile index f295452b..256f50b7 100644 --- a/Makefile +++ b/Makefile @@ -20,16 +20,16 @@ format: uv run black src/ test/ uv run ruff check --fix src/ test/ -# Run all tests +# Run all tests with combined coverage test: test-unit test-integration -# Run unit tests +# Run unit tests (creates .coverage report) test-unit: - uv run pytest test/ --ignore=test/integration/ --verbose + uv run pytest test/ --ignore=test/integration/ --verbose --cov=src/s3_encryption --cov-report=term-missing -# Run integration tests +# Run integration tests (appends to .coverage report from test-unit) test-integration: - uv run pytest test/integration/ --verbose + uv run pytest test/integration/ --verbose --cov=src/s3_encryption --cov-append --cov-report=term-missing # Clean up cache files clean: diff --git a/pyproject.toml b/pyproject.toml index a5ab41ef..93fcbfcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ [project.optional-dependencies] test = [ "pytest>=8.4.1", + "pytest-cov>=6.1.1", ] dev = [ "black>=24.3.0,<27.0.0", @@ -61,3 +62,9 @@ known-first-party = ["s3_encryption"] [tool.ruff.lint.per-file-ignores] "test/**/*.py" = ["D100", "D101", "D102", "D103", "D104", "E501"] "src/s3_encryption/pipelines.py" = ["E501"] + +[tool.coverage.run] +source = ["src/s3_encryption"] + +[tool.coverage.report] +show_missing = true From d022b983374a1d6c9addd748981ad21a65ef0730 Mon Sep 17 00:00:00 2001 From: Tony Knapp <5892063+texastony@users.noreply.github.com> Date: Fri, 27 Mar 2026 12:22:40 -0700 Subject: [PATCH 61/81] ci: daily ci with slack webhook (#162) --- .github/workflows/daily_ci.yml | 50 ++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/daily_ci.yml diff --git a/.github/workflows/daily_ci.yml b/.github/workflows/daily_ci.yml new file mode 100644 index 00000000..51e42fd4 --- /dev/null +++ b/.github/workflows/daily_ci.yml @@ -0,0 +1,50 @@ +name: Daily CI + +on: + schedule: + # 5 AM PST = 1 PM UTC, Monday–Friday + - cron: "0 13 * * 1-5" + workflow_dispatch: + inputs: + python-version: + description: 'Python version to use' + default: '3.11' + required: false + type: string + +jobs: + run-test-server: + permissions: + id-token: write + contents: read + name: Run TestServer Tests + uses: ./.github/workflows/test-server.yml + with: + python-version: ${{ inputs.python-version || '3.11' }} + secrets: inherit + + python-integ: + permissions: + id-token: write + contents: read + name: Python Integration Tests + uses: ./.github/workflows/python-integ.yml + with: + python-version: ${{ inputs.python-version || '3.11' }} + secrets: inherit + + notify: + needs: + [ + run-test-server, + python-integ + ] + permissions: + id-token: write + contents: read + if: ${{ failure() }} + uses: aws/aws-cryptographic-material-providers-library/.github/workflows/slack-notification.yml@main + with: + message: "Daily CI failed on `${{ github.repository }}`. View run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_CI }} From c46190955ffb8457d7d8a76676d3b1d89b06c10e Mon Sep 17 00:00:00 2001 From: Tony Knapp <5892063+texastony@users.noreply.github.com> Date: Thu, 2 Apr 2026 10:15:39 -0700 Subject: [PATCH 62/81] feat(decryption): streaming decryption with cipher-agnostic stream classes (#150) Add streaming decryption support via cipher-agnostic stream class, replacing eager in-memory decryption for all algorithm suites. Stream classes: - DecryptingStream: inherits from botocore.response.StreamingBody Decryptor classes: - Decryptor(ABC): AES Mode Metaclass - AesCbcDecryptor - AesGcmDecryptor Changes: - GetEncryptedObjectPipeline.decrypt() returns streaming decryptors for all algorithm suites (AES-GCM, key-committing AES-GCM/HKDF, AES-CBC) - Add cipher_tag_length_bytes and cipher_block_size_bytes to AlgorithmSuite, replacing hardcoded constants - CBC path always streams (no auth tag, matches Java S3EC behavior) --- src/s3_encryption/__init__.py | 54 +- src/s3_encryption/buffered_decrypt.py | 20 + src/s3_encryption/decryptor.py | 141 ++++ src/s3_encryption/materials/materials.py | 20 + src/s3_encryption/pipelines.py | 248 ++++--- src/s3_encryption/stream.py | 189 +++++ test/integration/test_i_s3_encryption.py | 23 + .../test_i_s3_encryption_instruction_file.py | 55 +- .../test_i_s3_encryption_streaming.py | 193 ++++++ test/test_decryption.py | 36 +- test/test_default_algorithm_commitment.py | 7 +- test/test_key_commitment.py | 26 +- test/test_pipelines.py | 29 +- test/test_s3_encryption_client_plugin.py | 15 + test/test_stream.py | 655 ++++++++++++++++++ 15 files changed, 1574 insertions(+), 137 deletions(-) create mode 100644 src/s3_encryption/buffered_decrypt.py create mode 100644 src/s3_encryption/decryptor.py create mode 100644 src/s3_encryption/stream.py create mode 100644 test/integration/test_i_s3_encryption_streaming.py create mode 100644 test/test_stream.py diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index f2af6d7a..9b8772d6 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -34,7 +34,29 @@ @define class S3EncryptionClientConfig: - """Configuration object for the S3 Encryption Client.""" + """Configuration for the S3 Encryption Client. + + Attributes: + keyring: Keyring used for encrypting/decrypting data keys. + encryption_algorithm: Algorithm suite for encryption. Defaults to + ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY (V3 key-committing). + commitment_policy: Key commitment policy for encryption and decryption. + Defaults to REQUIRE_ENCRYPT_REQUIRE_DECRYPT. + enable_legacy_unauthenticated_modes: If True, allow decryption of objects + encrypted with legacy CBC algorithm suites. Defaults to False. + cmm: Crypto materials manager. Defaults to a DefaultCryptoMaterialsManager + wrapping the provided keyring. + instruction_file_suffix: Suffix appended to the S3 object key when + fetching instruction files. Defaults to ".instruction". + enable_delayed_authentication: If True, release plaintext from streams + before GCM tag verification. Defaults to False. Has no effect for + CBC encrypted ciphertext, which is always streamed as there is no + authentication tag. + + Raises: + S3EncryptionClientError: If the encryption algorithm is legacy, or if + the algorithm suite is incompatible with the commitment policy. + """ keyring: AbstractKeyring encryption_algorithm: AlgorithmSuite = field( @@ -60,6 +82,15 @@ class S3EncryptionClientConfig: ##% as its associated object suffixed with ".instruction". instruction_file_suffix: str = field(default=".instruction") + ##= specification/s3-encryption/client.md#enable-delayed-authentication + ##= type=implementation + ##% The S3EC MUST support the option to enable or disable Delayed Authentication mode. + + ##= specification/s3-encryption/client.md#enable-delayed-authentication + ##= type=implication + ##% Delayed Authentication mode MUST be set to false by default. + enable_delayed_authentication: bool = field(default=False) + @cmm.default def _default_cmm_for_keyring(self): return DefaultCryptoMaterialsManager(self.keyring) @@ -197,10 +228,18 @@ def on_get_object_after_call(self, parsed, **kwargs): # The parsed response already has the Body as a StreamingBody # We need to read it, decrypt it, and replace it + # content_length is going to the cipher-text's content length + content_length = parsed.get("ContentLength") + if content_length is None: + obj_key = getattr(self._context, _CTX_KEY, None) + raise S3EncryptionClientError( + f"S3 response is missing ContentLength and is invalid. Key: {obj_key}" + ) # Create a response dict that matches what the pipeline expects response = { "Body": parsed.get("Body"), "Metadata": parsed.get("Metadata", {}), + "ContentLength": content_length, } # Create a pipeline and decrypt the data @@ -212,18 +251,15 @@ def on_get_object_after_call(self, parsed, **kwargs): ) decrypted_data = pipeline.decrypt( response, - encryption_context, + instruction_suffix=self.config.instruction_file_suffix, + enable_delayed_authentication=self.config.enable_delayed_authentication, + encryption_context=encryption_context, bucket=getattr(self._context, _CTX_BUCKET, None), key=getattr(self._context, _CTX_KEY, None), - instruction_suffix=self.config.instruction_file_suffix, ) - # Create a new streaming body with the decrypted data - stream = io.BytesIO(decrypted_data) - streaming_body = StreamingBody(stream, len(decrypted_data)) - - # Replace body with decrypted data - parsed["Body"] = streaming_body + # Replace body with decrypting stream + parsed["Body"] = decrypted_data def process_instruction_file(self, parsed): """Process instruction file in plaintext mode. diff --git a/src/s3_encryption/buffered_decrypt.py b/src/s3_encryption/buffered_decrypt.py new file mode 100644 index 00000000..6c305751 --- /dev/null +++ b/src/s3_encryption/buffered_decrypt.py @@ -0,0 +1,20 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""One Shot decryption into a buffer.""" + +from io import BytesIO + +from botocore.response import StreamingBody + +from s3_encryption.decryptor import Decryptor + + +def one_shot_decrypt(streaming_body: StreamingBody, decryptor: Decryptor): + """Decrypt a streaming object. + + Args: + streaming_body (object): A streaming object. + decryptor (Decryptor): Decryptor object. + """ + plaintext = decryptor.finalize(streaming_body.read()) + return StreamingBody(BytesIO(plaintext), len(plaintext)) diff --git a/src/s3_encryption/decryptor.py b/src/s3_encryption/decryptor.py new file mode 100644 index 00000000..e3d4eece --- /dev/null +++ b/src/s3_encryption/decryptor.py @@ -0,0 +1,141 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Decryptor abstractions for S3 Encryption Client.""" + +from abc import ABC, abstractmethod + +from attrs import define, field +from cryptography.exceptions import InvalidTag + +from .exceptions import S3EncryptionClientError, S3EncryptionClientSecurityError + + +class Decryptor(ABC): + """Abstract base class for content decryption. + + Implementations own all cipher and padding state, presenting a uniform + streaming interface to the decrypting stream classes. + """ + + @property + @abstractmethod + def content_length(self) -> int: + """Total byte length of the encrypted content (ciphertext + any trailing tag).""" + + @property + @abstractmethod + def amount_read(self) -> int: + """Number of ciphertext bytes consumed so far.""" + + @abstractmethod + def update(self, data: bytes) -> bytes: + """Process a chunk of ciphertext, returning any available plaintext.""" + + @abstractmethod + def finalize(self, data: bytes) -> bytes: + """Process the final chunk of ciphertext and finalize decryption.""" + + +@define +class AesCbcDecryptor(Decryptor): + """AES-CBC decryptor that owns both the cipher and PKCS7 unpadder. + + Args: + decryptor: A cryptography CBC cipher decryptor context. + unpadder: A cryptography PKCS7 unpadding context. + content_length: Total byte length of the CBC ciphertext. + """ + + _decryptor: object = field() + _unpadder: object = field() + _content_length: int = field() + _amount_read: int = field(init=False, default=0) + + @property + def content_length(self) -> int: # noqa: D102 + return self._content_length + + @property + def amount_read(self) -> int: # noqa: D102 + return self._amount_read + + def update(self, data: bytes) -> bytes: + """Decrypt a chunk and unpad incrementally.""" + self._amount_read += len(data) + plaintext = self._decryptor.update(data) + return self._unpadder.update(plaintext) + + def finalize(self, data: bytes) -> bytes: + """Finalize CBC decryption and flush the unpadder.""" + try: + self._amount_read += len(data) + plaintext = self._decryptor.update(data) if data else b"" + plaintext += self._decryptor.finalize() + return self._unpadder.update(plaintext) + self._unpadder.finalize() + except Exception as e: + raise S3EncryptionClientSecurityError(f"Failed to decrypt CBC content: {e}") from e + + +@define +class AesGcmDecryptor(Decryptor): + """AES-GCM decryptor that handles trailing auth tag verification. + + Args: + decryptor: A cryptography GCM cipher decryptor context. + tag_length: Length of the GCM authentication tag in bytes. + content_length: Total byte length of the encrypted content (ciphertext + tag). + """ + + _decryptor: object = field() + _tag_length: int = field() + _content_length: int = field() + _amount_read: int = field(init=False, default=0) + _tail: bytes = field(init=False, default=b"") + + @property + def content_length(self) -> int: # noqa: D102 + return self._content_length + + @property + def amount_read(self) -> int: # noqa: D102 + return self._amount_read + + @property + def tag_length(self) -> int: + """Length of the GCM authentication tag in bytes.""" + return self._tag_length + + def update(self, data: bytes) -> bytes: + """Decrypt a chunk, holding back the last tag_length bytes. + + A rolling _tail buffer always retains the last tag_length bytes + so the GCM tag is never passed to the cipher's update(). + """ + self._amount_read += len(data) + buf = self._tail + data + if len(buf) <= self._tag_length: + self._tail = buf + return b"" + self._tail = buf[-self._tag_length :] + return self._decryptor.update(buf[: -self._tag_length]) + + def finalize(self, data: bytes) -> bytes: + """Finalize decryption using the buffered tag.""" + try: + self._amount_read += len(data) + buf = self._tail + data + if len(buf) < self._tag_length: + raise S3EncryptionClientError( + f"Incomplete GCM data: expected at least {self._tag_length} " + f"tag bytes, got {len(buf)} total remaining bytes." + ) + tag = buf[-self._tag_length :] + ciphertext = buf[: -self._tag_length] + plaintext = self._decryptor.update(ciphertext) if ciphertext else b"" + return plaintext + self._decryptor.finalize_with_tag(tag) + except S3EncryptionClientError: + raise + except InvalidTag as e: + raise S3EncryptionClientSecurityError(f"Failed to decrypt Object: {e}") from e + except Exception as e: + raise S3EncryptionClientError(f"Failed to decrypt Object: {e}") from e diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index f2e8fd4f..80f682f0 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -172,6 +172,26 @@ def kc_gcm_iv(self) -> bytes: ##% the IV used in the AES-GCM content encryption/decryption MUST consist entirely of bytes with the value 0x01. return b"\x01" * self.cipher_iv_length_bytes + @property + def cipher_block_size_bits(self) -> int: + """Block size of the cipher in bits.""" + return self._cipher_block_size_bits + + @property + def cipher_block_size_bytes(self) -> int: + """Block size of the cipher in bytes.""" + return self._cipher_block_size_bits // 8 + + @property + def cipher_tag_length_bits(self) -> int: + """Authentication tag length of the cipher in bits.""" + return self._cipher_tag_length_bits + + @property + def cipher_tag_length_bytes(self) -> int: + """Authentication tag length of the cipher in bytes.""" + return self._cipher_tag_length_bits // 8 + class CommitmentPolicy(Enum): """Commitment policies controlling key-commitment behavior.""" diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index d0e9ba79..5561047a 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -11,11 +11,14 @@ import os from attrs import define, field +from botocore.response import StreamingBody from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cryptography.hazmat.primitives.ciphers.aead import AESGCM from cryptography.hazmat.primitives.padding import PKCS7 -from .exceptions import S3EncryptionClientError, S3EncryptionClientSecurityError +from .buffered_decrypt import one_shot_decrypt +from .decryptor import AesCbcDecryptor, AesGcmDecryptor +from .exceptions import S3EncryptionClientError from .instruction_file import fetch_instruction_file from .key_derivation import derive_keys, verify_commitment from .materials.crypto_materials_manager import AbstractCryptoMaterialsManager @@ -27,6 +30,7 @@ EncryptionMaterials, ) from .metadata import ObjectMetadata +from .stream import DecryptingStream @define @@ -222,26 +226,28 @@ def _determine_algorithm_suite(self, metadata) -> AlgorithmSuite: def decrypt( self, response, + instruction_suffix, + enable_delayed_authentication, encryption_context=None, bucket=None, key=None, - instruction_suffix=".instruction", - ): + ) -> StreamingBody: """Decrypt the data after it is retrieved from S3. Args: response (dict): The response from S3 containing the encrypted data and metadata + instruction_suffix (str): suffix for instruction file + enable_delayed_authentication (bool): If True, release plaintext before GCM tag verification. encryption_context (dict, optional): Additional context for decryption bucket (str, optional): S3 bucket name (required for instruction file) key (str, optional): S3 object key (required for instruction file) - instruction_suffix(str, optional): suffix for instruction file; defaults to ".instruction". Returns: - bytes: The decrypted data + A botocore.response.StreamingBody of plain-text """ # Convert the metadata dictionary to an ObjectMetadata instance - # TODO: Stream + Buffered Decryption - encrypted_data = response.get("Body").read() + streaming_body: StreamingBody = response.get("Body") + content_length = response.get("ContentLength") encryption_metadata = response.get("Metadata", {}) metadata = ObjectMetadata.from_dict(encryption_metadata) @@ -254,10 +260,12 @@ def decrypt( if self.s3_client is None: raise S3EncryptionClientError("s3_client required to fetch instruction file") - # TODO: we should validate that these parameters must be None - # when not in instruction file mode. if bucket is None or key is None: raise S3EncryptionClientError("Bucket and key required to fetch instruction file") + if instruction_suffix is None: + raise S3EncryptionClientError( + "instruction_suffix required to fetch instruction file" + ) instruction_key = key + instruction_suffix instruction_metadata = fetch_instruction_file(self.s3_client, bucket, instruction_key) @@ -380,24 +388,145 @@ def decrypt( ##= type=implementation ##% When the commitment policy is REQUIRE_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST allow decryption using algorithm suites which do not support key commitment. - # Perform decryption based on algorithm suite + if enable_delayed_authentication is None: + raise S3EncryptionClientError("enable_delayed_authentication must be explicitly set") + + # Build decryptor and return streaming wrapper based on algorithm suite match dec_materials.algorithm_suite: case AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF: - return self._decrypt_cbc_content(dec_materials, encrypted_data) + return self._decrypt_cbc_streaming(dec_materials, streaming_body, content_length) case AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: - ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf - ##= type=implementation - ##% The client MUST NOT provide any AAD when encrypting with - ##% ALG_AES_256_GCM_IV12_TAG16_NO_KDF. - aesgcm = AESGCM(dec_materials.plaintext_data_key) - return aesgcm.decrypt( - nonce=dec_materials.iv, data=encrypted_data, associated_data=None + return self._decrypt_gcm_streaming( + dec_materials, + streaming_body, + enable_delayed_authentication, + content_length, ) case AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: - return self._decrypt_kc_gcm_content(dec_materials, encrypted_data, metadata) + return self._decrypt_kc_gcm_streaming( + dec_materials, + metadata, + streaming_body, + enable_delayed_authentication, + content_length, + ) case _: raise S3EncryptionClientError("Unknown algorithm suite!") + @staticmethod + def _decrypt_cbc_streaming(dec_materials, streaming_body, content_length): + """Decrypt content encrypted with ALG_AES_256_CBC_IV16_NO_KDF. + + CBC is always streamed (no buffered mode) since it has no auth tag. + """ + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=implementation + ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and + ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is enabled, + ##% then the S3EC MUST create a cipher with AES in CBC Mode with PKCS5Padding or + ##% PKCS7Padding compatible padding for a 16-byte block cipher + ##% (example: for the Java JCE, this is "AES/CBC/PKCS5Padding"). + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=implementation + ##% If the cipher object cannot be created as described above, + ##% Decryption MUST fail. + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=implementation + ##% The error SHOULD detail why the cipher could not be initialized + ##% (such as CBC or PKCS5Padding is not supported by the underlying crypto provider). + cipher = Cipher( + algorithms.AES(dec_materials.plaintext_data_key), + modes.CBC(dec_materials.iv), + ) + # Remove PKCS7 padding (compatible with PKCS5Padding for 16-byte block ciphers) + unpadder = PKCS7(dec_materials.algorithm_suite.cipher_block_size_bits).unpadder() + decryptor = AesCbcDecryptor(cipher.decryptor(), unpadder, content_length=content_length) + return DecryptingStream(streaming_body, decryptor, content_length=content_length) + + @staticmethod + def _decrypt_gcm_streaming( + dec_materials, streaming_body, enable_delayed_authentication, content_length + ): + """Decrypt content encrypted with ALG_AES_256_GCM_IV12_TAG16_NO_KDF.""" + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf + ##= type=implementation + ##% The client MUST NOT provide any AAD when encrypting with + ##% ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + cipher = Cipher( + algorithms.AES(dec_materials.plaintext_data_key), modes.GCM(dec_materials.iv) + ) + decryptor = AesGcmDecryptor( + cipher.decryptor(), + tag_length=dec_materials.algorithm_suite.cipher_tag_length_bytes, + content_length=content_length, + ) + if enable_delayed_authentication: + ##= specification/s3-encryption/client.md#enable-delayed-authentication + ##= type=implementation + ##% When enabled, the S3EC MAY release plaintext from a stream which has not been authenticated. + return DecryptingStream(streaming_body, decryptor, content_length=content_length) + ##= specification/s3-encryption/client.md#enable-delayed-authentication + ##= type=implementation + ##% When disabled the S3EC MUST NOT release plaintext from a stream which has not been authenticated. + return one_shot_decrypt(streaming_body, decryptor) + + def _decrypt_kc_gcm_streaming( + self, dec_materials, metadata, streaming_body, enable_delayed_authentication, content_length + ): + """Decrypt content encrypted with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY. + + Performs HKDF key derivation, key commitment verification, then returns + a streaming decryptor. + """ + message_id = base64.b64decode(metadata.message_id_v3) + stored_commitment = base64.b64decode(metadata.key_commitment_v3) + + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##= type=implementation + ##% When using an algorithm suite which supports key commitment, the client MUST verify + ##% that the [derived key commitment](./key-derivation.md#hkdf-operation) contains the + ##% same bytes as the stored key commitment retrieved from the stored object's metadata. + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##= type=implementation + ##% When using an algorithm suite which supports key commitment, the client MUST verify the key commitment values match before deriving + ##% the [derived encryption key](./key-derivation.md#hkdf-operation). + derived_encryption_key, derived_commitment = derive_keys( + dec_materials.plaintext_data_key, message_id, dec_materials.algorithm_suite + ) + verify_commitment(stored_commitment, derived_commitment) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% When encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ##% the IV used in the AES-GCM content encryption/decryption MUST consist entirely of bytes with the value 0x01. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% The IV's total length MUST match the IV length defined by the algorithm suite. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% The client MUST initialize the cipher, or call an AES-GCM encryption API, with the derived encryption key, an IV containing only bytes with the value 0x01, + ##% and the tag length defined in the Algorithm Suite when encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. + cipher = Cipher( + algorithms.AES(derived_encryption_key), + modes.GCM(dec_materials.algorithm_suite.kc_gcm_iv), + ) + cipher_decryptor = cipher.decryptor() + cipher_decryptor.authenticate_additional_data(dec_materials.algorithm_suite.suite_id_bytes) + decryptor = AesGcmDecryptor( + cipher_decryptor, + tag_length=dec_materials.algorithm_suite.cipher_tag_length_bytes, + content_length=content_length, + ) + if enable_delayed_authentication: + return DecryptingStream(streaming_body, decryptor, content_length=content_length) + ##= specification/s3-encryption/client.md#enable-delayed-authentication + ##= type=implementation + ##% When disabled the S3EC MUST NOT release plaintext from a stream which has not been authenticated. + return one_shot_decrypt(streaming_body, decryptor) + def _decrypt_v2(self, metadata, encryption_context) -> DecryptionMaterials: """Prepare V2 decryption materials.""" return self._decrypt_v1_v2( @@ -440,40 +569,6 @@ def _decrypt_v1_v2( return self.cmm.decrypt_materials(dec_materials) - def _decrypt_cbc_content(self, dec_materials, encrypted_data): - """Decrypt content encrypted with ALG_AES_256_CBC_IV16_NO_KDF.""" - ##= specification/s3-encryption/decryption.md#cbc-decryption - ##= type=implementation - ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and - ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is enabled, - ##% then the S3EC MUST create a cipher with AES in CBC Mode with PKCS5Padding or - ##% PKCS7Padding compatible padding for a 16-byte block cipher - ##% (example: for the Java JCE, this is "AES/CBC/PKCS5Padding"). - ##= specification/s3-encryption/decryption.md#cbc-decryption - ##= type=implementation - ##% If the cipher object cannot be created as described above, - ##% Decryption MUST fail. - ##= specification/s3-encryption/decryption.md#cbc-decryption - ##= type=implementation - ##% The error SHOULD detail why the cipher could not be initialized - ##% (such as CBC or PKCS5Padding is not supported by the underlying crypto provider). - try: - cipher = Cipher( - algorithms.AES(dec_materials.plaintext_data_key), - modes.CBC(dec_materials.iv), - ) - decryptor = cipher.decryptor() - padded_plaintext = decryptor.update(encrypted_data) + decryptor.finalize() - - # Remove PKCS7 padding (compatible with PKCS5Padding for 16-byte block ciphers) - unpadder = PKCS7(128).unpadder() - return unpadder.update(padded_plaintext) + unpadder.finalize() - except Exception as e: - raise S3EncryptionClientSecurityError( - f"Failed to decrypt CBC content: {e}. " - "Ensure the underlying crypto provider supports AES/CBC/PKCS7Padding." - ) from e - ##= specification/s3-encryption/data-format/content-metadata.md#v3-only ##% The V3 format uses compression here such that each wrapping algorithm is represented by a two digit string. ##= specification/s3-encryption/data-format/content-metadata.md#v3-only @@ -527,50 +622,3 @@ def _decrypt_v3(self, metadata, encryption_context) -> DecryptionMaterials: ) return self.cmm.decrypt_materials(dec_materials) - - def _decrypt_kc_gcm_content(self, dec_materials, encrypted_data, metadata): - """Decrypt content encrypted with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY. - - Performs HKDF key derivation, key commitment verification, and AES-GCM decryption. - """ - message_id = base64.b64decode(metadata.message_id_v3) - stored_commitment = base64.b64decode(metadata.key_commitment_v3) - - ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key - ##= type=implementation - ##% The client MUST use HKDF to derive the key commitment value and the derived encrypting key as described in [Key Derivation](key-derivation.md). - derived_encryption_key, derived_commitment = derive_keys( - dec_materials.plaintext_data_key, message_id, dec_materials.algorithm_suite - ) - - ##= specification/s3-encryption/decryption.md#decrypting-with-commitment - ##= type=implementation - ##% When using an algorithm suite which supports key commitment, the client MUST verify - ##% that the [derived key commitment](./key-derivation.md#hkdf-operation) contains the - ##% same bytes as the stored key commitment retrieved from the stored object's metadata. - ##= specification/s3-encryption/decryption.md#decrypting-with-commitment - ##= type=implementation - ##% When using an algorithm suite which supports key commitment, the client MUST verify the key commitment values match before deriving - ##% the [derived encryption key](./key-derivation.md#hkdf-operation). - verify_commitment(stored_commitment, derived_commitment) - - ##= specification/s3-encryption/key-derivation.md#hkdf-operation - ##= type=implementation - ##% When encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - ##% the IV used in the AES-GCM content encryption/decryption MUST consist entirely of bytes with the value 0x01. - ##= specification/s3-encryption/key-derivation.md#hkdf-operation - ##= type=implementation - ##% The IV's total length MUST match the IV length defined by the algorithm suite. - ##= specification/s3-encryption/key-derivation.md#hkdf-operation - ##= type=implementation - ##% The client MUST initialize the cipher, or call an AES-GCM encryption API, with the derived encryption key, an IV containing only bytes with the value 0x01, - ##% and the tag length defined in the Algorithm Suite when encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY. - ##= specification/s3-encryption/key-derivation.md#hkdf-operation - ##= type=implementation - ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. - aesgcm = AESGCM(derived_encryption_key) - return aesgcm.decrypt( - nonce=dec_materials.algorithm_suite.kc_gcm_iv, - data=encrypted_data, - associated_data=dec_materials.algorithm_suite.suite_id_bytes, - ) diff --git a/src/s3_encryption/stream.py b/src/s3_encryption/stream.py new file mode 100644 index 00000000..a4e85a74 --- /dev/null +++ b/src/s3_encryption/stream.py @@ -0,0 +1,189 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Streaming decryption support for S3 Encryption Client.""" + +import io + +from attrs import define, field +from botocore.exceptions import IncompleteReadError +from botocore.response import StreamingBody + +from .decryptor import Decryptor + +##= specification/s3-encryption/client.md#set-buffer-size +##= type=exception +##= reason=Optional Feature that is a two-way door to implement later +##% The S3EC SHOULD accept a configurable buffer size which refers to the maximum ciphertext length in bytes to store in memory when Delayed Authentication mode is disabled. +##= specification/s3-encryption/client.md#set-buffer-size +##= type=exception +##= reason=Optional Feature that is a two-way door to implement later +##% If Delayed Authentication mode is enabled, and the buffer size has been set to a value other than its default, the S3EC MUST throw an exception. +##= specification/s3-encryption/client.md#set-buffer-size +##= type=exception +##= reason=Optional Feature that is a two-way door to implement later +##% If Delayed Authentication mode is disabled, and no buffer size is provided, the S3EC MUST set the buffer size to a reasonable default. + + +_DEFAULT_CHUNK_SIZE = 1024 + + +##= specification/s3-encryption/client.md#enable-delayed-authentication +##= type=implementation +##% When enabled, the S3EC MAY release plaintext from a stream which has not been authenticated. +# slots=False because StreamingBody extends IOBase which already has __weakref__. +@define(slots=False) +class DecryptingStream(StreamingBody): + """A stream that releases plaintext incrementally before full authentication. + + Extends botocore's StreamingBody so it can be used as a drop-in replacement + for parsed["Body"]. All StreamingBody methods are explicitly overridden. + """ + + # This stream is ALMOST cipher-agnostic — the Decryptor handles ALMOST all algorithm details. + # Ciphertext is fed through decryptor.update() incrementally, and + # decryptor.finalize() is called with any trailing data when the body is exhausted. + # + # ALMOST :: The AES-GCM tag is problematic when combined with iterators that can split + # the tag over two reads. To accommodate this, read() has a while loop with 3 return conditions. + # See inline comments of read for more details. + + _body: object = field() + _decryptor: Decryptor = field() + _content_length: int = field() + _bytes_consumed: int = field(init=False, default=0) + _finalized: bool = field(init=False, default=False) + + def __attrs_post_init__(self): # noqa: D105 + super().__init__(io.BytesIO(), content_length=self._content_length) + + def readable(self): # noqa: D102 + return not self._finalized + + def read(self, amt=None): + """Read and decrypt ciphertext, releasing plaintext incrementally. + + Args: + amt: Number of bytes to read. If None, reads all remaining data. + + Returns: + bytes: Decrypted plaintext bytes. + """ + if self._finalized: + return b"" + + # Loop until the decryptor produces non-empty plaintext. + # The GCM decryptor's tail buffer may absorb small reads entirely + # (returning b"" from update) while it holds back the trailing auth + # tag. Looping prevents callers from seeing spurious empty bytes + # mid-stream, which would break `while chunk := stream.read(amt)`. + result = b"" + while not result: + remaining = self._content_length - self._bytes_consumed + if remaining <= 0: + # All content_length bytes consumed — finalize with no extra data. + return self._finalize(b"") + + # Never read past content_length; cap at amt if provided. + to_read = remaining if amt is None else min(amt, remaining) + raw = self._body.read(to_read) + + if not raw: + # Underlying stream exhausted early — finalize with what we have. + return self._finalize(b"") + + self._bytes_consumed += len(raw) + remaining = self._content_length - self._bytes_consumed + + if remaining <= 0: + # This is the last chunk — pass it to finalize so the decryptor + # can split off the GCM tag (or flush CBC padding) and verify. + return self._finalize(raw) + + # Feed ciphertext to the decryptor. For GCM, the tail buffer holds + # back the last tag_length bytes, so update() may return b"" if + # the chunk was entirely absorbed into the buffer. + result = self._decryptor.update(raw) + return result + + def _finalize(self, trailing_data): + """Finalize decryption with any trailing data.""" + if self._finalized: + return b"" + self._finalized = True + plaintext = self._decryptor.finalize(trailing_data) + self._verify_content_length() + return plaintext + + def readinto(self, b): + """Read bytes into a pre-allocated, writable bytes-like object b. + + Returns the number of bytes decrypted. + Note: CBC Padding and GCM tag will be removed, so bytes read MAYBE greater than bytes decrypted. + """ + data = self.read(len(b)) + n = len(data) + b[:n] = data + return n + + def readlines(self): # noqa: D102 + return self.read().splitlines(True) + + def __iter__(self): + """Return an iterator to yield 1k chunks from the decryption stream.""" + return self + + def __next__(self): + """Return the next 1k chunk from the decryption stream.""" + chunk = self.read(_DEFAULT_CHUNK_SIZE) + if chunk: + return chunk + raise StopIteration() + + next = __next__ + + def iter_lines(self, chunk_size=_DEFAULT_CHUNK_SIZE, keepends=False): + """Return an iterator to yield lines from the decryption stream. + + This is achieved by reading chunk of bytes (of size chunk_size) at a + time from the chipher-text stream, decrypting them, and then yielding lines from there. + """ + pending = b"" + for chunk in self.iter_chunks(chunk_size): + lines = (pending + chunk).splitlines(True) + for line in lines[:-1]: + yield line.splitlines(keepends)[0] + pending = lines[-1] + if pending: + yield pending.splitlines(keepends)[0] + + def iter_chunks(self, chunk_size=_DEFAULT_CHUNK_SIZE): + """Return an iterator to yield chunks of chunk_size bytes from the raw stream.""" + while True: + chunk = self.read(chunk_size) + if chunk == b"": + break + yield chunk + + def _verify_content_length(self): + """Verify that the decryptor consumed exactly content_length bytes.""" + if self._decryptor.content_length is not None and not ( + self._decryptor.amount_read == self._content_length + ): + raise IncompleteReadError( + actual_bytes=self._decryptor.amount_read, + expected_bytes=self._decryptor.content_length, + ) + + def tell(self): # noqa: D102 + return self._bytes_consumed + + def close(self): + """Close the underlying cipher-text stream.""" + if hasattr(self._body, "close"): + self._body.close() + + def __enter__(self): # noqa: D105 + return self + + def __exit__(self, *args): # noqa: D105 + self.close() diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index 15133c05..36f826bd 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -252,3 +252,26 @@ def test_put_object_uses_configured_algorithm(algorithm_suite, commitment_policy meta_key, expected_value = _EXPECTED_ALGORITHM_METADATA[algorithm_suite] assert meta_key in metadata, f"Expected metadata key '{meta_key}' not found in {metadata}" assert metadata[meta_key] == expected_value + + +##= specification/s3-encryption/client.md#enable-delayed-authentication +##= type=test +##% The S3EC MUST support the option to enable or disable Delayed Authentication mode. +@pytest.mark.parametrize("enable_delayed_auth", [False, True], ids=["buffered", "delayed-auth"]) +def test_delayed_authentication_mode(enable_delayed_auth): + """S3EC MUST support enabling and disabling delayed authentication.""" + key = _unique_key("delayed-auth-mode-") + data = b"test delayed authentication mode" + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig( + keyring, + enable_delayed_authentication=enable_delayed_auth, + ) + s3ec = S3EncryptionClient(wrapped_client, config) + + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == data diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index f4f70704..6c93d832 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -24,6 +24,8 @@ "v2_instruction_file": "static-v2-instruction-file-from-java-v4", "v3_instruction_file": "static-v3-instruction-file-from-java-v4", "negative_v2_instruction_file": "NEGATIVE-static-v2-instruction-file-test-from-java-v4", + "large_v2_instruction_file": "static-large-v2-instruction-file-from-java-v4-52428800", + "large_v3_instruction_file": "static-large-v3-instruction-file-from-java-v4-52428800", } @@ -53,7 +55,8 @@ def test_decrypt_v1_instruction_file(): print("Success! V1 instruction file decryption completed.") -def test_decrypt_v2_instruction_file(): +@pytest.mark.parametrize("delayed_auth", [False, True], ids=["buffered", "delayed-auth"]) +def test_decrypt_v2_instruction_file(delayed_auth): """Test decrypting V2 object with instruction file. V2 format uses ALG_AES_256_GCM_IV12_TAG16_NO_KDF (no key commitment). @@ -68,6 +71,7 @@ def test_decrypt_v2_instruction_file(): keyring, encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + enable_delayed_authentication=delayed_auth, ) s3ec = S3EncryptionClient(wrapped_client, config) @@ -145,7 +149,8 @@ def test_decrypt_v3_instruction_file_custom_suffix(): print("Success! V3 custom suffix instruction file decryption completed.") -def test_decrypt_v2_instruction_file_custom_suffix(): +@pytest.mark.parametrize("delayed_auth", [False, True], ids=["buffered", "delayed-auth"]) +def test_decrypt_v2_instruction_file_custom_suffix(delayed_auth): """Test decrypting V2 object with a custom instruction file suffix.""" key = TEST_OBJECTS["v2_instruction_file"] @@ -157,6 +162,7 @@ def test_decrypt_v2_instruction_file_custom_suffix(): encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, instruction_file_suffix=".custom-suffix-instruction", commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + enable_delayed_authentication=delayed_auth, ) s3ec = S3EncryptionClient(wrapped_client, config) @@ -165,3 +171,48 @@ def test_decrypt_v2_instruction_file_custom_suffix(): assert output == "static-v2-instruction-file-from-java-v4" print("Success! V2 custom suffix instruction file decryption completed.") + + +LARGE_FILE_SIZE = 52428800 # 50 MB + + +def test_decrypt_large_v2_instruction_file_delayed_auth(): + """Test streaming decryption of a 50 MB V2 object with delayed authentication.""" + key = TEST_OBJECTS["large_v2_instruction_file"] + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig( + keyring, + enable_delayed_authentication=True, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) + s3ec = S3EncryptionClient(wrapped_client, config) + + response = s3ec.get_object(Bucket=bucket, Key=key) + total = 0 + while chunk := response["Body"].read(65536): + total += len(chunk) + + assert total == LARGE_FILE_SIZE + + +@pytest.mark.skip(reason="V3 large file not yet written to static bucket") +def test_decrypt_large_v3_instruction_file_delayed_auth(): + """Test streaming decryption of a 50 MB V3 object with delayed authentication.""" + key = TEST_OBJECTS["large_v3_instruction_file"] + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring, enable_delayed_authentication=True) + s3ec = S3EncryptionClient(wrapped_client, config) + + response = s3ec.get_object(Bucket=bucket, Key=key) + total = 0 + while chunk := response["Body"].read(65536): + total += len(chunk) + + assert total == LARGE_FILE_SIZE diff --git a/test/integration/test_i_s3_encryption_streaming.py b/test/integration/test_i_s3_encryption_streaming.py new file mode 100644 index 00000000..530959bb --- /dev/null +++ b/test/integration/test_i_s3_encryption_streaming.py @@ -0,0 +1,193 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Integration tests for streaming decryption modes (buffered vs delayed-auth). + +These tests verify that BufferedDecryptingGCMStream and DelayedAuthGCMDecryptingStream +produce correct plaintext for real S3 round-trips across algorithm suites. +""" + +import os +from datetime import datetime + +import boto3 +import pytest +from botocore.response import StreamingBody + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy +from s3_encryption.stream import DecryptingStream + +bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") +region = os.environ.get("CI_AWS_REGION", "us-west-2") +kms_key_id = os.environ.get( + "CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" +) + +GCM_CONFIGS = [ + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + id="AES_GCM", + ), + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + id="KC_GCM", + ), +] + + +def _make_client(algorithm_suite, commitment_policy, delayed_auth): + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig( + keyring, + encryption_algorithm=algorithm_suite, + commitment_policy=commitment_policy, + enable_delayed_authentication=delayed_auth, + ) + return S3EncryptionClient(wrapped_client, config) + + +def _unique_key(prefix): + return prefix + datetime.now().strftime("%Y-%m-%d-%H:%M:%S-%f") + + +# --------------------------------------------------------------------------- +# Buffered mode: verifies tag before releasing plaintext +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", GCM_CONFIGS) +def test_buffered_roundtrip(algorithm_suite, commitment_policy): + """Buffered mode decrypts correctly for a simple round-trip.""" + key = _unique_key("buffered-rt-") + data = b"buffered mode round trip test data" + + s3ec = _make_client(algorithm_suite, commitment_policy, delayed_auth=False) + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + response = s3ec.get_object(Bucket=bucket, Key=key) + + body = response["Body"] + assert isinstance(body, StreamingBody) + assert body.read() == data + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", GCM_CONFIGS) +def test_buffered_partial_reads(algorithm_suite, commitment_policy): + """Buffered mode supports partial read(amt) calls.""" + key = _unique_key("buffered-partial-") + data = os.urandom(1024) + + s3ec = _make_client(algorithm_suite, commitment_policy, delayed_auth=False) + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + response = s3ec.get_object(Bucket=bucket, Key=key) + + result = b"" + while chunk := response["Body"].read(100): + result += chunk + assert result == data + + +# --------------------------------------------------------------------------- +# Delayed-auth mode: releases plaintext before tag verification +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", GCM_CONFIGS) +def test_delayed_auth_roundtrip(algorithm_suite, commitment_policy): + """Delayed-auth mode decrypts correctly for a simple round-trip.""" + key = _unique_key("delayed-rt-") + data = b"delayed auth round trip test data" + + s3ec = _make_client(algorithm_suite, commitment_policy, delayed_auth=True) + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + response = s3ec.get_object(Bucket=bucket, Key=key) + + body = response["Body"] + assert isinstance(body, DecryptingStream) + assert body.read() == data + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", GCM_CONFIGS) +def test_delayed_auth_chunked_reads(algorithm_suite, commitment_policy): + """Delayed-auth mode supports chunked streaming reads.""" + key = _unique_key("delayed-chunked-") + data = os.urandom(4096) + + s3ec = _make_client(algorithm_suite, commitment_policy, delayed_auth=True) + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + response = s3ec.get_object(Bucket=bucket, Key=key) + + result = b"" + while chunk := response["Body"].read(256): + result += chunk + assert result == data + + +# --------------------------------------------------------------------------- +# Both modes produce identical plaintext +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", GCM_CONFIGS) +def test_buffered_and_delayed_produce_same_plaintext(algorithm_suite, commitment_policy): + """Both streaming modes must produce identical plaintext for the same object.""" + key = _unique_key("same-plaintext-") + data = os.urandom(2048) + + # Encrypt once + writer = _make_client(algorithm_suite, commitment_policy, delayed_auth=False) + writer.put_object(Bucket=bucket, Key=key, Body=data) + + # Decrypt with buffered + buffered = _make_client(algorithm_suite, commitment_policy, delayed_auth=False) + resp_buf = buffered.get_object(Bucket=bucket, Key=key) + plaintext_buf = resp_buf["Body"].read() + + # Decrypt with delayed-auth + delayed = _make_client(algorithm_suite, commitment_policy, delayed_auth=True) + resp_del = delayed.get_object(Bucket=bucket, Key=key) + plaintext_del = resp_del["Body"].read() + + assert plaintext_buf == plaintext_del == data + + +# --------------------------------------------------------------------------- +# Empty body +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("delayed_auth", [False, True], ids=["buffered", "delayed-auth"]) +@pytest.mark.parametrize("algorithm_suite,commitment_policy", GCM_CONFIGS) +def test_empty_body_roundtrip(algorithm_suite, commitment_policy, delayed_auth): + """Both modes handle empty plaintext correctly.""" + key = _unique_key("empty-stream-") + + s3ec = _make_client(algorithm_suite, commitment_policy, delayed_auth=delayed_auth) + s3ec.put_object(Bucket=bucket, Key=key, Body=b"") + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == b"" + + +# --------------------------------------------------------------------------- +# Large object streaming +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", GCM_CONFIGS) +def test_delayed_auth_large_object(algorithm_suite, commitment_policy): + """Delayed-auth streams a 1 MB object correctly via chunked reads.""" + key = _unique_key("delayed-large-") + data = os.urandom(1024 * 1024) # 1 MB + + s3ec = _make_client(algorithm_suite, commitment_policy, delayed_auth=True) + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + response = s3ec.get_object(Bucket=bucket, Key=key) + + result = b"" + while chunk := response["Body"].read(65536): + result += chunk + assert result == data diff --git a/test/test_decryption.py b/test/test_decryption.py index 6d22d439..7f9a51d1 100644 --- a/test/test_decryption.py +++ b/test/test_decryption.py @@ -74,7 +74,7 @@ def _v2_gcm_metadata(): def _response(metadata, body=b"ciphertext"): - return {"Body": BytesIO(body), "Metadata": metadata} + return {"Body": BytesIO(body), "Metadata": metadata, "ContentLength": len(body)} # --------------------------------------------------------------------------- @@ -106,7 +106,9 @@ def test_cbc_object_rejected_when_legacy_disabled(self): ) with pytest.raises(S3EncryptionClientError, match="ALG_AES_256_CBC_IV16_NO_KDF"): - pipeline.decrypt(_response(_v1_cbc_metadata())) + pipeline.decrypt( + _response(_v1_cbc_metadata()), ".instruction", enable_delayed_authentication=False + ) ##= specification/s3-encryption/decryption.md#cbc-decryption ##= type=test @@ -148,8 +150,10 @@ def test_cbc_decryption_succeeds_when_legacy_enabled(self): keyring_return=dec_mats, ) - result = pipeline.decrypt(_response(metadata, ciphertext)) - assert result == plaintext + result = pipeline.decrypt( + _response(metadata, ciphertext), ".instruction", enable_delayed_authentication=False + ) + assert result.read() == plaintext ##= specification/s3-encryption/decryption.md#cbc-decryption ##= type=test @@ -193,7 +197,9 @@ def test_cbc_decryption_fails_with_wrong_key(self): ) with pytest.raises(S3EncryptionClientSecurityError, match="Failed to decrypt CBC content"): - pipeline.decrypt(_response(metadata, ciphertext)) + pipeline.decrypt( + _response(metadata, ciphertext), ".instruction", enable_delayed_authentication=False + ).read() # --------------------------------------------------------------------------- @@ -289,7 +295,11 @@ def test_commitment_verified_before_content_decryption(self): with pytest.raises( S3EncryptionClientSecurityError, match="Key commitment verification failed" ): - pipeline.decrypt(_response(metadata, b"fake-ciphertext")) + pipeline.decrypt( + _response(metadata, b"fake-ciphertext"), + ".instruction", + enable_delayed_authentication=False, + ) # --------------------------------------------------------------------------- @@ -322,7 +332,9 @@ def test_require_decrypt_rejects_non_committing_suite(self): ) with pytest.raises(S3EncryptionClientError, match="cannot decrypt non-key-committing"): - pipeline.decrypt(_response(_v2_gcm_metadata())) + pipeline.decrypt( + _response(_v2_gcm_metadata()), ".instruction", enable_delayed_authentication=False + ) def test_allow_decrypt_accepts_non_committing_suite(self): """REQUIRE_ENCRYPT_ALLOW_DECRYPT MUST allow non-committing algorithm suites.""" @@ -352,8 +364,10 @@ def test_allow_decrypt_accepts_non_committing_suite(self): keyring_return=dec_mats, ) - result = pipeline.decrypt(_response(metadata, ciphertext)) - assert result == plaintext + result = pipeline.decrypt( + _response(metadata, ciphertext), ".instruction", enable_delayed_authentication=False + ) + assert result.read() == plaintext # --------------------------------------------------------------------------- @@ -387,4 +401,6 @@ def test_legacy_cbc_rejected_by_default(self): ) with pytest.raises(S3EncryptionClientError, match="not configured to decrypt"): - pipeline.decrypt(_response(_v1_cbc_metadata())) + pipeline.decrypt( + _response(_v1_cbc_metadata()), ".instruction", enable_delayed_authentication=False + ) diff --git a/test/test_default_algorithm_commitment.py b/test/test_default_algorithm_commitment.py index 1640451a..7e61ed7f 100644 --- a/test/test_default_algorithm_commitment.py +++ b/test/test_default_algorithm_commitment.py @@ -83,6 +83,7 @@ def test_default_encryption_decryptable_with_require_decrypt(self): response = { "Body": BytesIO(ciphertext), "Metadata": metadata, + "ContentLength": len(ciphertext), } # Decrypt with REQUIRE_ENCRYPT_REQUIRE_DECRYPT — this will reject @@ -91,5 +92,7 @@ def test_default_encryption_decryptable_with_require_decrypt(self): cmm, commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, ) - result = decrypt_pipeline.decrypt(response) - assert result == plaintext + result = decrypt_pipeline.decrypt( + response, ".instruction", enable_delayed_authentication=False + ) + assert result.read() == plaintext diff --git a/test/test_key_commitment.py b/test/test_key_commitment.py index a82fc9fd..a0be12ce 100644 --- a/test/test_key_commitment.py +++ b/test/test_key_commitment.py @@ -66,7 +66,11 @@ def _v2_gcm_response(key, plaintext=b"test data"): plaintext_data_key=key, algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, ) - return {"Body": BytesIO(ciphertext), "Metadata": metadata}, dec_mats, plaintext + return ( + {"Body": BytesIO(ciphertext), "Metadata": metadata, "ContentLength": len(ciphertext)}, + dec_mats, + plaintext, + ) def _v3_kc_gcm_response(key, plaintext=b"test data"): @@ -88,7 +92,11 @@ def _v3_kc_gcm_response(key, plaintext=b"test data"): plaintext_data_key=key, algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, ) - return {"Body": BytesIO(ciphertext), "Metadata": metadata}, dec_mats, plaintext + return ( + {"Body": BytesIO(ciphertext), "Metadata": metadata, "ContentLength": len(ciphertext)}, + dec_mats, + plaintext, + ) # --------------------------------------------------------------------------- @@ -111,8 +119,8 @@ def test_forbid_encrypt_allows_non_committing_decrypt(self): commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, keyring_return=dec_mats, ) - result = pipeline.decrypt(response) - assert result == plaintext + result = pipeline.decrypt(response, ".instruction", enable_delayed_authentication=False) + assert result.read() == plaintext ##= specification/s3-encryption/key-commitment.md#commitment-policy ##= type=test @@ -126,8 +134,8 @@ def test_require_encrypt_allow_decrypt_allows_non_committing_decrypt(self): commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, keyring_return=dec_mats, ) - result = pipeline.decrypt(response) - assert result == plaintext + result = pipeline.decrypt(response, ".instruction", enable_delayed_authentication=False) + assert result.read() == plaintext ##= specification/s3-encryption/key-commitment.md#commitment-policy ##= type=test @@ -142,7 +150,7 @@ def test_require_require_rejects_non_committing_decrypt(self): keyring_return=dec_mats, ) with pytest.raises(S3EncryptionClientError, match="cannot decrypt non-key-committing"): - pipeline.decrypt(response) + pipeline.decrypt(response, ".instruction", enable_delayed_authentication=False) def test_require_require_allows_committing_decrypt(self): """REQUIRE_ENCRYPT_REQUIRE_DECRYPT MUST allow decryption with committing suites.""" @@ -153,5 +161,5 @@ def test_require_require_allows_committing_decrypt(self): commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, keyring_return=dec_mats, ) - result = pipeline.decrypt(response) - assert result == plaintext + result = pipeline.decrypt(response, ".instruction", enable_delayed_authentication=False) + assert result.read() == plaintext diff --git a/test/test_pipelines.py b/test/test_pipelines.py index 3d32b8cc..e3d34e35 100644 --- a/test/test_pipelines.py +++ b/test/test_pipelines.py @@ -66,7 +66,13 @@ def test_decrypt_v1_from_instruction_file(self): # Should fail when trying to decrypt (proving instruction file was fetched) with pytest.raises(Exception, match="Keyring called"): - pipeline.decrypt(mock_response, bucket="test-bucket", key="test-key") + pipeline.decrypt( + mock_response, + instruction_suffix=".instruction", + enable_delayed_authentication=False, + bucket="test-bucket", + key="test-key", + ) # Verify instruction file was fetched mock_s3_client.get_object.assert_called_once_with( @@ -125,7 +131,13 @@ def test_decrypt_v2_from_instruction_file(self): # Should fail when trying to decrypt (proving instruction file was fetched) with pytest.raises(Exception, match="Keyring called"): - pipeline.decrypt(mock_response, bucket="test-bucket", key="test-key") + pipeline.decrypt( + mock_response, + instruction_suffix=".instruction", + enable_delayed_authentication=False, + bucket="test-bucket", + key="test-key", + ) # Verify instruction file was fetched mock_s3_client.get_object.assert_called_once_with( @@ -199,7 +211,13 @@ def test_decrypt_v3_from_instruction_file(self): with pytest.raises( S3EncryptionClientSecurityError, match="Key commitment verification failed" ): - pipeline.decrypt(mock_response, bucket="test-bucket", key="test-key") + pipeline.decrypt( + mock_response, + instruction_suffix=".instruction", + enable_delayed_authentication=False, + bucket="test-bucket", + key="test-key", + ) # Verify instruction file was fetched mock_s3_client.get_object.assert_called_once_with( @@ -250,9 +268,10 @@ def test_decrypt_with_custom_instruction_file_suffix(self): with pytest.raises(Exception, match="Keyring called"): pipeline.decrypt( mock_response, + instruction_suffix=".custom-suffix", + enable_delayed_authentication=False, bucket="test-bucket", key="test-key", - instruction_suffix=".custom-suffix", ) mock_s3_client.get_object.assert_called_once_with( @@ -290,4 +309,4 @@ def test_decrypt_v3_unsupported_wrap_alg(self): with pytest.raises( S3EncryptionClientError, match="AES/GCM is not a valid key wrapping algorithm" ): - pipeline.decrypt(mock_response) + pipeline.decrypt(mock_response, ".instruction", enable_delayed_authentication=False) diff --git a/test/test_s3_encryption_client_plugin.py b/test/test_s3_encryption_client_plugin.py index bdc48c79..cbc8cd80 100644 --- a/test/test_s3_encryption_client_plugin.py +++ b/test/test_s3_encryption_client_plugin.py @@ -139,3 +139,18 @@ def test_instruction_file_mode_invalid_keys_raises_error(self): # Should raise error with pytest.raises(S3EncryptionClientError, match="Instruction file contains invalid keys"): plugin.on_get_object_after_call(parsed) + + def test_missing_content_length_raises_error(self): + """Test that a missing ContentLength in the S3 response raises an error.""" + mock_keyring = Mock(spec=S3Keyring) + config = S3EncryptionClientConfig(keyring=mock_keyring) + plugin = S3EncryptionClientPlugin(config) + plugin._context.key = "my-object" + + parsed = { + "Body": StreamingBody(io.BytesIO(b"data"), 4), + "Metadata": {}, + } + + with pytest.raises(S3EncryptionClientError, match="missing ContentLength.*Key: my-object"): + plugin.on_get_object_after_call(parsed) diff --git a/test/test_stream.py b/test/test_stream.py new file mode 100644 index 00000000..dc0f2eb9 --- /dev/null +++ b/test/test_stream.py @@ -0,0 +1,655 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for streaming decryption behavior.""" + +import os +from io import BytesIO +from unittest.mock import Mock + +import pytest +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.padding import PKCS7 + +from s3_encryption.buffered_decrypt import one_shot_decrypt +from s3_encryption.decryptor import AesCbcDecryptor, AesGcmDecryptor +from s3_encryption.exceptions import S3EncryptionClientSecurityError +from s3_encryption.stream import DecryptingStream + + +def _encrypt_gcm(plaintext: bytes): + """Encrypt plaintext with AES-GCM, return (ciphertext_with_tag, key, nonce).""" + key = os.urandom(32) + nonce = os.urandom(12) + ciphertext_with_tag = AESGCM(key).encrypt(nonce, plaintext, None) + return ciphertext_with_tag, key, nonce + + +def _make_gcm_decryptor(key, nonce, content_length): + """Create an AesGcmDecryptor.""" + cipher_decryptor = Cipher(algorithms.AES(key), modes.GCM(nonce)).decryptor() + return AesGcmDecryptor(cipher_decryptor, tag_length=16, content_length=content_length) + + +def _encrypt_cbc(plaintext: bytes): + """Encrypt plaintext with AES-CBC + PKCS7 padding, return (ciphertext, key, iv).""" + key = os.urandom(32) + iv = os.urandom(16) + padder = PKCS7(128).padder() + padded = padder.update(plaintext) + padder.finalize() + encryptor = Cipher(algorithms.AES(key), modes.CBC(iv)).encryptor() + ciphertext = encryptor.update(padded) + encryptor.finalize() + return ciphertext, key, iv + + +def _make_cbc_decryptor(key, iv, content_length): + """Create an AesCbcDecryptor.""" + cipher_decryptor = Cipher(algorithms.AES(key), modes.CBC(iv)).decryptor() + unpadder = PKCS7(128).unpadder() + return AesCbcDecryptor(cipher_decryptor, unpadder, content_length=content_length) + + +def _make_streaming_body(data: bytes): + """Create a mock StreamingBody wrapping data.""" + body = Mock() + stream = BytesIO(data) + body.read = stream.read + body.close = Mock() + body._stream = stream + return body + + +class TestDelayedAuthReleasesBeforeVerification: + """Delayed auth releases plaintext before the GCM tag is verified.""" + + ##= specification/s3-encryption/client.md#enable-delayed-authentication + ##= type=test + ##% When enabled, the S3EC MAY release plaintext from a stream which has not been authenticated. + def test_delayed_auth_releases_plaintext_before_tag_verification(self): + plaintext = os.urandom(4096) + ct, key, nonce = _encrypt_gcm(plaintext) + body = _make_streaming_body(ct) + + stream = DecryptingStream( + body, + _make_gcm_decryptor(key, nonce, len(ct)), + content_length=len(ct), + ) + # read(256) decrypts a partial chunk via cipher.update(), releasing + # plaintext without consuming the full ciphertext stream. The GCM tag + # at the end of the stream has not been reached yet. + chunk = stream.read(256) + + # Plaintext was returned before the stream was fully consumed + assert len(chunk) > 0 + # _finalized is False: the GCM tag has NOT been verified yet + assert not stream._finalized + # Ciphertext remains unread in the underlying stream + assert body._stream.tell() < len(ct) + + # Finish reading the stream and verify full plaintext matches + remaining = stream.read() + assert chunk + remaining == plaintext + + +class TestBufferedWithholdsUntilVerification: + """Buffered mode does not release plaintext until the GCM tag is verified.""" + + ##= specification/s3-encryption/client.md#enable-delayed-authentication + ##= type=test + ##% When disabled the S3EC MUST NOT release plaintext from a stream which has not been authenticated. + def test_buffered_verifies_tag_before_releasing_any_plaintext(self): + plaintext = os.urandom(4096) + ct, key, nonce = _encrypt_gcm(plaintext) + body = _make_streaming_body(ct) + + decryptor = _make_gcm_decryptor(key, nonce, len(ct)) + original_finalize = decryptor.finalize + finalize_called = [] + + def spy_finalize(data): + result = original_finalize(data) + finalize_called.append(True) + return result + + decryptor.finalize = spy_finalize + + stream = one_shot_decrypt(body, decryptor) + + # one_shot_decrypt calls finalize() eagerly — tag is verified + # before any read() call on the returned stream. + assert finalize_called, "finalize (tag verification) must happen before read()" + chunk = stream.read(1) + assert chunk == plaintext[:1] + + +class TestDelayedAuthCBCDecryption: + + def test_roundtrip(self): + plaintext = b"hello world, this is a CBC test!!" + ciphertext, key, iv = _encrypt_cbc(plaintext) + stream = DecryptingStream( + _make_streaming_body(ciphertext), + _make_cbc_decryptor(key, iv, len(ciphertext)), + content_length=len(ciphertext), + ) + assert stream.read() == plaintext + + def test_chunked_read(self): + plaintext = b"A" * 256 + ciphertext, key, iv = _encrypt_cbc(plaintext) + stream = DecryptingStream( + _make_streaming_body(ciphertext), + _make_cbc_decryptor(key, iv, len(ciphertext)), + content_length=len(ciphertext), + ) + result = b"" + while chunk := stream.read(64): + result += chunk + assert result == plaintext + + def test_finalize_called(self): + plaintext = b"finalize me" + ciphertext, key, iv = _encrypt_cbc(plaintext) + stream = DecryptingStream( + _make_streaming_body(ciphertext), + _make_cbc_decryptor(key, iv, len(ciphertext)), + content_length=len(ciphertext), + ) + actual = stream.read() + assert stream._finalized + assert actual == plaintext + + def test_no_trailing_padding_bytes(self): + plaintext = b"short" + ciphertext, key, iv = _encrypt_cbc(plaintext) + stream = DecryptingStream( + _make_streaming_body(ciphertext), + _make_cbc_decryptor(key, iv, len(ciphertext)), + content_length=len(ciphertext), + ) + assert stream.read() == plaintext + + def test_read_after_finalized_returns_empty(self): + plaintext = b"done" + ciphertext, key, iv = _encrypt_cbc(plaintext) + stream = DecryptingStream( + _make_streaming_body(ciphertext), + _make_cbc_decryptor(key, iv, len(ciphertext)), + content_length=len(ciphertext), + ) + stream.read() + assert stream.read() == b"" + + def test_readable_false_after_finalized(self): + plaintext = b"readable" + ciphertext, key, iv = _encrypt_cbc(plaintext) + stream = DecryptingStream( + _make_streaming_body(ciphertext), + _make_cbc_decryptor(key, iv, len(ciphertext)), + content_length=len(ciphertext), + ) + assert stream.readable() + actual = stream.read() + assert not stream.readable() + assert actual == plaintext + + def test_close_delegates_to_body(self): + plaintext = b"close me" + ciphertext, key, iv = _encrypt_cbc(plaintext) + body = _make_streaming_body(ciphertext) + stream = DecryptingStream( + body, + _make_cbc_decryptor(key, iv, len(ciphertext)), + content_length=len(ciphertext), + ) + stream.close() + body.close.assert_called_once() + + def test_enter_returns_self(self): + plaintext = b"ctx" + ciphertext, key, iv = _encrypt_cbc(plaintext) + stream = DecryptingStream( + _make_streaming_body(ciphertext), + _make_cbc_decryptor(key, iv, len(ciphertext)), + content_length=len(ciphertext), + ) + assert stream.__enter__() is stream + + def test_wrong_key_raises_error(self): + plaintext = b"wrong key test!!" + ciphertext, _key, iv = _encrypt_cbc(plaintext) + wrong_key = os.urandom(32) + stream = DecryptingStream( + _make_streaming_body(ciphertext), + _make_cbc_decryptor(wrong_key, iv, len(ciphertext)), + content_length=len(ciphertext), + ) + with pytest.raises(S3EncryptionClientSecurityError, match="Failed to decrypt CBC content"): + stream.read() + + def test_empty_ciphertext(self): + key = os.urandom(32) + iv = os.urandom(16) + stream = DecryptingStream( + _make_streaming_body(b""), + _make_cbc_decryptor(key, iv, 0), + content_length=0, + ) + # Empty stream finalize will fail because CBC expects at least one block + with pytest.raises(S3EncryptionClientSecurityError, match="Failed to decrypt CBC content"): + stream.read() + + +class TestBufferedDecryptingStream: + + def test_full_read(self): + plaintext = os.urandom(1024) + ct, key, nonce = _encrypt_gcm(plaintext) + stream = one_shot_decrypt( + _make_streaming_body(ct), _make_gcm_decryptor(key, nonce, len(ct)) + ) + assert stream.read() == plaintext + + def test_partial_reads(self): + plaintext = os.urandom(512) + ct, key, nonce = _encrypt_gcm(plaintext) + stream = one_shot_decrypt( + _make_streaming_body(ct), + _make_gcm_decryptor(key, nonce, len(ct)), + ) + result = b"" + while chunk := stream.read(100): + result += chunk + assert result == plaintext + + def test_read_triggers_full_decrypt(self): + plaintext = os.urandom(256) + ct, key, nonce = _encrypt_gcm(plaintext) + body = _make_streaming_body(ct) + decryptor = _make_gcm_decryptor(key, nonce, len(ct)) + finalize_called = [] + original_finalize = decryptor.finalize + decryptor.finalize = lambda data: (finalize_called.append(True), original_finalize(data))[1] + + stream = one_shot_decrypt(body, decryptor) + # one_shot_decrypt eagerly decrypts — finalize called at construction + assert finalize_called + # Entire ciphertext consumed from the body + assert body._stream.tell() == len(ct) + assert stream.read(1) == plaintext[:1] + + def test_tell(self): + plaintext = os.urandom(200) + ct, key, nonce = _encrypt_gcm(plaintext) + stream = one_shot_decrypt( + _make_streaming_body(ct), + _make_gcm_decryptor(key, nonce, len(ct)), + ) + stream.read(50) + assert stream.tell() == 50 + + def test_readable(self): + plaintext = b"readable test" + ct, key, nonce = _encrypt_gcm(plaintext) + stream = one_shot_decrypt( + _make_streaming_body(ct), + _make_gcm_decryptor(key, nonce, len(ct)), + ) + assert stream.readable() + + def test_readinto(self): + """Asserts that readinto is implemented.""" + plaintext = os.urandom(64) + ct, key, nonce = _encrypt_gcm(plaintext) + stream = one_shot_decrypt( + _make_streaming_body(ct), + _make_gcm_decryptor(key, nonce, len(ct)), + ) + buf = bytearray(64) + n = stream.readinto(buf) + assert n == 64 + assert bytes(buf) == plaintext + + def test_enter_returns_stream(self): + plaintext = b"enter" + ct, key, nonce = _encrypt_gcm(plaintext) + stream = one_shot_decrypt( + _make_streaming_body(ct), + _make_gcm_decryptor(key, nonce, len(ct)), + ) + with stream as s: + assert s.read() == plaintext + + def test_close(self): + """Asserts that close does not raise.""" + plaintext = b"close" + ct, key, nonce = _encrypt_gcm(plaintext) + body = _make_streaming_body(ct) + stream = one_shot_decrypt( + body, + _make_gcm_decryptor(key, nonce, len(ct)), + ) + stream.close() # should not raise + + def test_close_without_close_attr(self): + """Asserts that close handles bodies without close.""" + plaintext = b"no close" + ct, key, nonce = _encrypt_gcm(plaintext) + body = Mock() + del body.close + body.read = BytesIO(ct).read + stream = one_shot_decrypt( + body, + _make_gcm_decryptor(key, nonce, len(ct)), + ) + stream.close() # should not raise + + def test_wrong_key_raises_error(self): + plaintext = b"wrong key" + ct, _key, nonce = _encrypt_gcm(plaintext) + wrong_key = os.urandom(32) + with pytest.raises(S3EncryptionClientSecurityError, match="Failed to decrypt"): + one_shot_decrypt( + _make_streaming_body(ct), + _make_gcm_decryptor(wrong_key, nonce, len(ct)), + ) + + def test_tampered_ciphertext_raises_error(self): + plaintext = b"tamper test" + ct, key, nonce = _encrypt_gcm(plaintext) + tampered = bytearray(ct) + tampered[0] ^= 0xFF + with pytest.raises(S3EncryptionClientSecurityError, match="Failed to decrypt"): + one_shot_decrypt( + _make_streaming_body(bytes(tampered)), + _make_gcm_decryptor(key, nonce, len(ct)), + ) + + def test_idempotent_decrypt(self): + plaintext = os.urandom(128) + ct, key, nonce = _encrypt_gcm(plaintext) + stream = one_shot_decrypt( + _make_streaming_body(ct), + _make_gcm_decryptor(key, nonce, len(ct)), + ) + first = stream.read(63) + second = stream.read(65) + assert first + second == plaintext + + +class TestDelayedAuthGCMDecryption: + + def test_full_read(self): + plaintext = os.urandom(1024) + ct, key, nonce = _encrypt_gcm(plaintext) + stream = DecryptingStream( + _make_streaming_body(ct), + _make_gcm_decryptor(key, nonce, len(ct)), + content_length=len(ct), + ) + assert stream.read() == plaintext + + def test_chunked_read(self): + plaintext = os.urandom(512) + ct, key, nonce = _encrypt_gcm(plaintext) + stream = DecryptingStream( + _make_streaming_body(ct), + _make_gcm_decryptor(key, nonce, len(ct)), + content_length=len(ct), + ) + result = b"" + while chunk := stream.read(64): + result += chunk + assert result == plaintext + + def test_read_after_finalized_returns_empty(self): + plaintext = os.urandom(128) + ct, key, nonce = _encrypt_gcm(plaintext) + stream = DecryptingStream( + _make_streaming_body(ct), + _make_gcm_decryptor(key, nonce, len(ct)), + content_length=len(ct), + ) + actual = stream.read() + assert stream._finalized + assert stream.read() == b"" + assert actual == plaintext + + def test_readable_false_after_finalized(self): + plaintext = b"readable" + ct, key, nonce = _encrypt_gcm(plaintext) + stream = DecryptingStream( + _make_streaming_body(ct), + _make_gcm_decryptor(key, nonce, len(ct)), + content_length=len(ct), + ) + assert stream.readable() + stream.read() + assert not stream.readable() + + def test_close_delegates(self): + plaintext = b"close" + ct, key, nonce = _encrypt_gcm(plaintext) + body = _make_streaming_body(ct) + stream = DecryptingStream( + body, + _make_gcm_decryptor(key, nonce, len(ct)), + content_length=len(ct), + ) + stream.close() + body.close.assert_called_once() + + def test_enter_returns_self(self): + plaintext = b"ctx" + ct, key, nonce = _encrypt_gcm(plaintext) + stream = DecryptingStream( + _make_streaming_body(ct), + _make_gcm_decryptor(key, nonce, len(ct)), + content_length=len(ct), + ) + assert stream.__enter__() is stream + + def test_wrong_key_raises_error(self): + plaintext = b"wrong key" + ct, _key, nonce = _encrypt_gcm(plaintext) + wrong_key = os.urandom(32) + stream = DecryptingStream( + _make_streaming_body(ct), + _make_gcm_decryptor(wrong_key, nonce, len(ct)), + content_length=len(ct), + ) + with pytest.raises(S3EncryptionClientSecurityError, match="Failed to decrypt"): + stream.read() + + def test_tampered_tag_raises_error(self): + plaintext = b"tamper tag" + ct, key, nonce = _encrypt_gcm(plaintext) + tampered = bytearray(ct) + tampered[-1] ^= 0xFF # flip last byte (part of tag) + stream = DecryptingStream( + _make_streaming_body(bytes(tampered)), + _make_gcm_decryptor(key, nonce, len(ct)), + content_length=len(ct), + ) + with pytest.raises(S3EncryptionClientSecurityError, match="Failed to decrypt"): + stream.read() + + def test_small_data_less_than_tag_length(self): + """Data exactly equal to tag length — only tag, no ciphertext.""" + plaintext = b"" + ct, key, nonce = _encrypt_gcm(plaintext) + # For empty plaintext, ct is just the 16-byte tag + assert len(ct) == 16 + stream = DecryptingStream( + _make_streaming_body(ct), + _make_gcm_decryptor(key, nonce, len(ct)), + content_length=len(ct), + ) + assert stream.read() == b"" + + def test_large_data(self): + plaintext = os.urandom(1024 * 1024) # 1 MB + ct, key, nonce = _encrypt_gcm(plaintext) + stream = DecryptingStream( + _make_streaming_body(ct), + _make_gcm_decryptor(key, nonce, len(ct)), + content_length=len(ct), + ) + result = b"" + while chunk := stream.read(65536): + result += chunk + assert result == plaintext + + +# --------------------------------------------------------------------------- +# Parameterized edge-case plaintext lengths +# --------------------------------------------------------------------------- +# Lengths chosen around AES block size (16) and two-block (32) boundaries, +# plus zero and one byte, to exercise padding, tag-splitting, and empty-data paths. +EDGE_CASE_LENGTHS = [0, 1, 8, 15, 16, 17, 31, 32, 33, 47, 48, 49, 300] + + +class TestEdgeCasePlaintextLengths: + + @pytest.mark.parametrize("length", EDGE_CASE_LENGTHS) + def test_buffered_gcm(self, length): + plaintext = os.urandom(length) + ct, key, nonce = _encrypt_gcm(plaintext) + stream = one_shot_decrypt( + _make_streaming_body(ct), + _make_gcm_decryptor(key, nonce, len(ct)), + ) + assert stream.read() == plaintext + + @pytest.mark.parametrize("length", EDGE_CASE_LENGTHS) + def test_delayed_auth_gcm(self, length): + plaintext = os.urandom(length) + ct, key, nonce = _encrypt_gcm(plaintext) + stream = DecryptingStream( + _make_streaming_body(ct), + _make_gcm_decryptor(key, nonce, len(ct)), + content_length=len(ct), + ) + result = b"" + while chunk := stream.read(7): + result += chunk + assert result == plaintext + + @pytest.mark.parametrize("length", EDGE_CASE_LENGTHS) + def test_delayed_auth_cbc(self, length): + plaintext = os.urandom(length) + ciphertext, key, iv = _encrypt_cbc(plaintext) + stream = DecryptingStream( + _make_streaming_body(ciphertext), + _make_cbc_decryptor(key, iv, len(ciphertext)), + content_length=len(ciphertext), + ) + result = b"" + while chunk := stream.read(7): + result += chunk + assert result == plaintext + + +class TestDecryptingStreamIterators: + """Tests for iter_chunks, iter_lines, __iter__, __next__, readinto, and readlines.""" + + def _make_gcm_stream(self, plaintext): + ct, key, nonce = _encrypt_gcm(plaintext) + return DecryptingStream( + _make_streaming_body(ct), + _make_gcm_decryptor(key, nonce, len(ct)), + content_length=len(ct), + ) + + @pytest.mark.parametrize("chunk_size", EDGE_CASE_LENGTHS[1:]) + def test_iter_chunks(self, chunk_size): + plaintext = os.urandom(300) + stream = self._make_gcm_stream(plaintext) + result = b"" + for chunk in stream.iter_chunks(chunk_size): + assert ( + len(chunk) <= chunk_size or not result + ) # first chunk may vary due to GCM buffering + result += chunk + assert result == plaintext + + def test_iter_chunks_default_size(self): + plaintext = os.urandom(2048) + stream = self._make_gcm_stream(plaintext) + result = b"".join(stream.iter_chunks()) + assert result == plaintext + + def test_iter_chunks_empty(self): + stream = self._make_gcm_stream(b"") + assert list(stream.iter_chunks()) == [] + + def test_iter(self): + plaintext = os.urandom(2048) + stream = self._make_gcm_stream(plaintext) + result = b"".join(stream) + assert result == plaintext + + def test_next(self): + plaintext = os.urandom(100) + stream = self._make_gcm_stream(plaintext) + first = next(stream) + assert len(first) > 0 + # drain the rest + rest = b"" + for chunk in stream: + rest += chunk + assert first + rest == plaintext + + def test_next_raises_stop_iteration(self): + stream = self._make_gcm_stream(b"") + with pytest.raises(StopIteration): + next(stream) + + def test_iter_lines(self): + plaintext = b"line1\nline2\nline3\n" + stream = self._make_gcm_stream(plaintext) + lines = list(stream.iter_lines()) + assert lines == [b"line1", b"line2", b"line3"] + + def test_iter_lines_keepends(self): + plaintext = b"line1\nline2\nline3\n" + stream = self._make_gcm_stream(plaintext) + lines = list(stream.iter_lines(keepends=True)) + assert lines == [b"line1\n", b"line2\n", b"line3\n"] + + def test_iter_lines_no_trailing_newline(self): + plaintext = b"first\nsecond" + stream = self._make_gcm_stream(plaintext) + lines = list(stream.iter_lines()) + assert lines == [b"first", b"second"] + + def test_iter_lines_empty(self): + stream = self._make_gcm_stream(b"") + assert list(stream.iter_lines()) == [] + + def test_readinto(self): + plaintext = os.urandom(64) + stream = self._make_gcm_stream(plaintext) + buf = bytearray(64) + n = stream.readinto(buf) + assert bytes(buf[:n]) == plaintext[:n] + + def test_readinto_partial(self): + plaintext = os.urandom(200) + stream = self._make_gcm_stream(plaintext) + buf = bytearray(50) + result = b"" + while n := stream.readinto(buf): + result += bytes(buf[:n]) + assert result == plaintext + + def test_readlines(self): + plaintext = b"aaa\nbbb\nccc\n" + stream = self._make_gcm_stream(plaintext) + assert stream.readlines() == [b"aaa\n", b"bbb\n", b"ccc\n"] + + def test_readlines_no_trailing_newline(self): + plaintext = b"aaa\nbbb" + stream = self._make_gcm_stream(plaintext) + assert stream.readlines() == [b"aaa\n", b"bbb"] From 52d05f3f9bb8ff0f66e78c3cb97d253eb0314cd4 Mon Sep 17 00:00:00 2001 From: Tony Knapp <5892063+texastony@users.noreply.github.com> Date: Mon, 6 Apr 2026 14:53:33 -0700 Subject: [PATCH 63/81] fix(error-handling): improve error messages for missing S3 objects and instruction files (#149) Match Java S3EC behavior when S3 objects or instruction files do not exist: - Add early return in event handler when Body is None (failed S3 response) - Catch ClientError separately in get_object with "Unable to retrieve object" message - Catch ClientError in fetch_instruction_file with instruction-file-specific message - Check for None body in process_instruction_file before reading Tests: - Unit: NoSuchKey, AccessDenied, and missing instruction file error wrapping - Integration: non-existent object and plain object with missing instruction file --- src/s3_encryption/__init__.py | 26 ++++++- src/s3_encryption/instruction_file.py | 8 +++ .../test_i_s3_encryption_instruction_file.py | 54 ++++++++++++++ test/test_s3_encryption_client.py | 72 +++++++++++++++++++ 4 files changed, 157 insertions(+), 3 deletions(-) create mode 100644 test/test_s3_encryption_client.py diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 9b8772d6..7ece424c 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -6,6 +6,7 @@ import threading from attrs import define, field +from botocore.exceptions import ClientError from botocore.response import StreamingBody from .exceptions import S3EncryptionClientError @@ -225,6 +226,11 @@ def on_get_object_after_call(self, parsed, **kwargs): # Get encryption context from thread-local storage (set by get_object wrapper) encryption_context = getattr(self._context, _CTX_ENCRYPTION_CONTEXT, None) + # If Body is None, the S3 request failed (e.g., NoSuchKey). + # Return early and let boto3 raise the original error. + if parsed.get("Body", None) is None: + return + # The parsed response already has the Body as a StreamingBody # We need to read it, decrypt it, and replace it @@ -272,9 +278,16 @@ def process_instruction_file(self, parsed): """ instruction_key = getattr(self._context, _CTX_KEY, None) + body = parsed.get("Body", None) + if body is None: + raise S3EncryptionClientError( + f"Instruction file body is empty for key: {instruction_key}" + ) + # In plaintext mode, parse instruction file and append to metadata - existing_metadata = parsed.get("Metadata", {}) - instruction_data = parsed.get("Body").read() + # Metadata may be present but None, so `or {}` handles that case + existing_metadata = parsed.get("Metadata", {}) or {} + instruction_data = body.read() instruction_metadata = parse_instruction_file(instruction_data, instruction_key) # Append parsed instruction file content to existing metadata @@ -385,9 +398,16 @@ def get_object(self, **kwargs): except S3EncryptionClientError: # Re-raise our own exceptions without wrapping raise + except ClientError as e: + # Wrap S3 service errors (e.g., NoSuchKey) with context + raise S3EncryptionClientError( + f"Failed to retrieve and/or decrypt object: {str(e)}" + ) from e except Exception as e: # Wrap any unexpected errors during decryption - raise S3EncryptionClientError(f"Failed to decrypt object: {str(e)}") from e + raise S3EncryptionClientError( + f"Failed to retrieve and/or decrypt object: {str(e)}" + ) from e finally: # Clean up thread-local storage; # do not clean up the client as it is not thread local only diff --git a/src/s3_encryption/instruction_file.py b/src/s3_encryption/instruction_file.py index 351c4b15..60305d17 100644 --- a/src/s3_encryption/instruction_file.py +++ b/src/s3_encryption/instruction_file.py @@ -9,6 +9,8 @@ import json from typing import Any +from botocore.exceptions import ClientError + from .exceptions import S3EncryptionClientError from .metadata import VALID_S3EC_METADATA_KEYS @@ -95,6 +97,12 @@ def fetch_instruction_file(s3_client, bucket: str, key: str) -> dict[str, Any]: try: response = s3_client.get_object(Bucket=bucket, Key=key) + except ClientError as e: + raise S3EncryptionClientError( + f"Exception encountered while fetching Instruction File. " + f"Ensure the object you are attempting to decrypt has been encrypted using the S3 Encryption Client. " + f"Instruction key: {key}" + ) from e finally: # Clear the flags after the call if hasattr(s3_client, "_s3ec_plugin_context"): diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index 6c93d832..6ca72e8b 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -1,6 +1,7 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 import os +import uuid import boto3 import pytest @@ -173,6 +174,58 @@ def test_decrypt_v2_instruction_file_custom_suffix(delayed_auth): print("Success! V2 custom suffix instruction file decryption completed.") +def test_get_nonexistent_object_raises_s3_encryption_client_error(): + """Test that getting a non-existent object raises S3EncryptionClientError. + + Matches Java S3EC behavior: NoSuchKeyException is wrapped in + S3EncryptionClientException with the original as the cause. + """ + from botocore.exceptions import ClientError + + from s3_encryption.exceptions import S3EncryptionClientError + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + with pytest.raises( + S3EncryptionClientError, match="Failed to retrieve and/or decrypt object" + ) as exc_info: + s3ec.get_object(Bucket=bucket, Key="this-object-does-not-exist") + + assert isinstance(exc_info.value.__cause__, ClientError) + + +def test_get_object_with_missing_instruction_file_raises_s3_encryption_client_error(): + """Test that a missing instruction file raises S3EncryptionClientError. + + When an object has no encryption metadata and the instruction file + also doesn't exist, the error should indicate the instruction file issue. + """ + from s3_encryption.exceptions import S3EncryptionClientError + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + # Use a separate plain S3 client to put an unencrypted object + plain_s3 = boto3.client("s3") + test_key = f"plain-object-no-instruction-file-{uuid.uuid4()}" + plain_s3.put_object(Bucket=bucket, Key=test_key, Body=b"hello") + + try: + with pytest.raises(S3EncryptionClientError, match="Instruction file body is empty"): + s3ec.get_object(Bucket=bucket, Key=test_key) + finally: + plain_s3.delete_object(Bucket=bucket, Key=test_key) + + LARGE_FILE_SIZE = 52428800 # 50 MB @@ -183,6 +236,7 @@ def test_decrypt_large_v2_instruction_file_delayed_auth(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig( keyring, enable_delayed_authentication=True, diff --git a/test/test_s3_encryption_client.py b/test/test_s3_encryption_client.py new file mode 100644 index 00000000..2b164fe8 --- /dev/null +++ b/test/test_s3_encryption_client.py @@ -0,0 +1,72 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for S3EncryptionClient get_object error handling.""" + +from unittest.mock import Mock + +import pytest +from botocore.exceptions import ClientError + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.materials.keyring import S3Keyring + + +class TestGetObjectNonExistentObject: + """S3EncryptionClient wraps S3 errors with context, preserving the original cause.""" + + def _build_client(self): + mock_s3 = Mock() + mock_s3.meta.events = Mock() + mock_s3.meta.events.register = Mock() + mock_keyring = Mock(spec=S3Keyring) + config = S3EncryptionClientConfig(keyring=mock_keyring) + return S3EncryptionClient(wrapped_s3_client=mock_s3, config=config), mock_s3 + + def test_no_such_key_raises_s3_encryption_client_error(self): + client, mock_s3 = self._build_client() + error_response = { + "Error": {"Code": "NoSuchKey", "Message": "The specified key does not exist."} + } + mock_s3.get_object.side_effect = ClientError(error_response, "GetObject") + + with pytest.raises( + S3EncryptionClientError, match="Failed to retrieve and/or decrypt object" + ) as exc_info: + client.get_object(Bucket="test-bucket", Key="nonexistent-key") + + assert isinstance(exc_info.value.__cause__, ClientError) + assert exc_info.value.__cause__.response["Error"]["Code"] == "NoSuchKey" + + def test_access_denied_raises_s3_encryption_client_error(self): + client, mock_s3 = self._build_client() + error_response = {"Error": {"Code": "AccessDenied", "Message": "Access Denied"}} + mock_s3.get_object.side_effect = ClientError(error_response, "GetObject") + + with pytest.raises( + S3EncryptionClientError, match="Failed to retrieve and/or decrypt object" + ) as exc_info: + client.get_object(Bucket="test-bucket", Key="forbidden-key") + + assert isinstance(exc_info.value.__cause__, ClientError) + assert exc_info.value.__cause__.response["Error"]["Code"] == "AccessDenied" + + +class TestFetchMissingInstructionFile: + """fetch_instruction_file wraps NoSuchKey with instruction-file-specific message.""" + + def test_missing_instruction_file_raises_s3_encryption_client_error(self): + mock_s3 = Mock() + mock_s3._s3ec_plugin_context = Mock() + error_response = { + "Error": {"Code": "NoSuchKey", "Message": "The specified key does not exist."} + } + mock_s3.get_object.side_effect = ClientError(error_response, "GetObject") + + from s3_encryption.instruction_file import fetch_instruction_file + + with pytest.raises(S3EncryptionClientError, match="fetching Instruction File") as exc_info: + fetch_instruction_file(mock_s3, "test-bucket", "test-key.instruction") + + assert isinstance(exc_info.value.__cause__, ClientError) + assert exc_info.value.__cause__.response["Error"]["Code"] == "NoSuchKey" From 09f3b1e0d6f15374a78f2488789b083febc482b7 Mon Sep 17 00:00:00 2001 From: Tony Knapp <5892063+texastony@users.noreply.github.com> Date: Thu, 16 Apr 2026 11:19:39 -0700 Subject: [PATCH 64/81] feat(examples): add usage examples with integration tests (#156) feat(examples): add usage examples with integration tests: - KMS Keyring put/get roundtrip with encryption context - Legacy V1 object decryption with enable_legacy_wrapping_algorithms - Delayed authentication streaming decryption for large files - Instruction file decryption with default and custom suffixes - Register examples pytest mark in pyproject.toml - Add examples step to CI workflow Also moves instruction_file_suffix from client-level config to a per-request get_object kwarg, allowing different suffixes per request. --- .github/workflows/python-integ.yml | 3 + Makefile | 5 +- examples/__init__.py | 2 + examples/src/__init__.py | 2 + .../src/delayed_auth_streaming_example.py | 89 +++++++++++++++++ examples/src/instruction_file_example.py | 59 ++++++++++++ examples/src/kms_keyring_put_get_example.py | 95 +++++++++++++++++++ examples/src/legacy_decrypt_example.py | 60 ++++++++++++ examples/test/__init__.py | 2 + .../test_i_delayed_auth_streaming_example.py | 27 ++++++ .../test/test_i_instruction_file_example.py | 27 ++++++ .../test_i_kms_keyring_put_get_example.py | 27 ++++++ .../test/test_i_legacy_decrypt_example.py | 27 ++++++ pyproject.toml | 5 + src/s3_encryption/__init__.py | 35 ++++--- src/s3_encryption/pipelines.py | 3 +- .../test_i_s3_encryption_instruction_file.py | 10 +- 17 files changed, 458 insertions(+), 20 deletions(-) create mode 100644 examples/__init__.py create mode 100644 examples/src/__init__.py create mode 100644 examples/src/delayed_auth_streaming_example.py create mode 100644 examples/src/instruction_file_example.py create mode 100644 examples/src/kms_keyring_put_get_example.py create mode 100644 examples/src/legacy_decrypt_example.py create mode 100644 examples/test/__init__.py create mode 100644 examples/test/test_i_delayed_auth_streaming_example.py create mode 100644 examples/test/test_i_instruction_file_example.py create mode 100644 examples/test/test_i_kms_keyring_put_get_example.py create mode 100644 examples/test/test_i_legacy_decrypt_example.py diff --git a/.github/workflows/python-integ.yml b/.github/workflows/python-integ.yml index b845e725..d2761518 100644 --- a/.github/workflows/python-integ.yml +++ b/.github/workflows/python-integ.yml @@ -56,6 +56,9 @@ jobs: CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} + - name: Run examples + run: make test-examples + - name: Generate coverage HTML report if: always() run: uv run coverage html -d coverage-report diff --git a/Makefile b/Makefile index 256f50b7..5960117a 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ format: uv run ruff check --fix src/ test/ # Run all tests with combined coverage -test: test-unit test-integration +test: test-unit test-integration test-examples # Run unit tests (creates .coverage report) test-unit: @@ -31,6 +31,9 @@ test-unit: test-integration: uv run pytest test/integration/ --verbose --cov=src/s3_encryption --cov-append --cov-report=term-missing +test-examples: + uv run pytest examples/test/ -v + # Clean up cache files clean: find . -type d -name __pycache__ -exec rm -rf {} + diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 00000000..f94fd12a --- /dev/null +++ b/examples/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/examples/src/__init__.py b/examples/src/__init__.py new file mode 100644 index 00000000..f94fd12a --- /dev/null +++ b/examples/src/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/examples/src/delayed_auth_streaming_example.py b/examples/src/delayed_auth_streaming_example.py new file mode 100644 index 00000000..5be4eb76 --- /dev/null +++ b/examples/src/delayed_auth_streaming_example.py @@ -0,0 +1,89 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +This example demonstrates streaming decryption with delayed authentication +using the S3 Encryption Client. + +By default, the S3 Encryption Client buffers the entire ciphertext and verifies +the authentication tag before releasing any plaintext. This is the safest mode, +but requires holding the entire object in memory. + +With delayed authentication enabled, plaintext is released incrementally as it +is decrypted, before the authentication tag has been verified. This allows +processing large files without buffering the entire object in memory. + +Your application should still read the stream to completion. In the event that +an error is thrown in the final read due to an invalid authentication tag, +your application must be able to invalidate the associated data. + +WARNING: With delayed authentication, plaintext is released before it has been +authenticated. An attacker could modify the ciphertext and the client would +release tampered plaintext before detecting the modification. Only use this +mode when you need to process files too large to buffer in memory and you +understand the security implications. + +This example: +1. Creates a KMS Keyring +2. Configures the S3 Encryption Client with delayed authentication enabled +3. Encrypts and uploads a large object to S3 +4. Streams the decrypted object back, reading it in chunks +5. Verifies the decrypted content matches the original +""" + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.exceptions import S3EncryptionClientSecurityError +from s3_encryption.materials.kms_keyring import KmsKeyring + +# 10 MB of example data +EXAMPLE_DATA: bytes = b"A" * (10 * 1024 * 1024) +CHUNK_SIZE = 1024 * 1024 # 1 MB + + +def delayed_auth_streaming_decrypt( + s3_client, kms_client, kms_key_id: str, bucket: str, key: str +): + """Demonstrate streaming decryption with delayed authentication. + + Args: + s3_client: boto3 S3 client. + kms_client: boto3 KMS client. + kms_key_id: KMS key ARN or alias to use for encryption/decryption. + bucket: S3 bucket name. + key: S3 object key. + """ + # 1. Create a KMS Keyring. + keyring = KmsKeyring(kms_client=kms_client, kms_key_id=kms_key_id) + + # 2. Configure the S3 Encryption Client with delayed authentication. + config = S3EncryptionClientConfig( + keyring=keyring, + enable_delayed_authentication=True, + ) + s3ec = S3EncryptionClient(wrapped_s3_client=s3_client, config=config) + + # 3. Encrypt and upload the object. + s3ec.put_object(Bucket=bucket, Key=key, Body=EXAMPLE_DATA) + + # 4. Stream the decrypted object back in chunks. + # With delayed authentication, plaintext is released incrementally + # without buffering the entire object in memory. + response = s3ec.get_object(Bucket=bucket, Key=key) + body = response["Body"] + + chunks = [] + try: + while True: + chunk = body.read(CHUNK_SIZE) + if not chunk: + break + chunks.append(chunk) + + plaintext = b"".join(chunks) + + except S3EncryptionClientSecurityError: + # Authentication tag verification failed. + # Discard any plaintext released before the error. + raise + + # 5. Verify the decrypted content matches the original. + assert plaintext == EXAMPLE_DATA, "Decrypted plaintext does not match original data" diff --git a/examples/src/instruction_file_example.py b/examples/src/instruction_file_example.py new file mode 100644 index 00000000..3c5db625 --- /dev/null +++ b/examples/src/instruction_file_example.py @@ -0,0 +1,59 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +This example demonstrates decrypting S3 objects that store their encryption +metadata in instruction files rather than S3 object metadata. + +An instruction file is a companion S3 object that contains the encryption +metadata (encrypted data key, IV, algorithm, etc.) as JSON. By default, +the instruction file has the same key as the encrypted object with a +".instruction" suffix appended. + +You can also use a custom instruction file suffix. This requires configuring +the S3 Encryption Client with the matching suffix. + +NOTE: At this time, the S3 Encryption Client in Python ONLY supports decrypting +(reading) with instruction files; encrypting with instruction files is not supported +at this time. + +This example: +1. Decrypts an object using the default instruction file suffix (".instruction") +2. Decrypts the same object using a custom instruction file suffix +""" + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.materials.kms_keyring import KmsKeyring + + +def instruction_file_get( + s3_client, kms_client, kms_key_id: str, bucket: str, key: str, expected_plaintext: bytes +): + """Demonstrate decrypting objects with default and custom instruction file suffixes. + + Args: + s3_client: boto3 S3 client. + kms_client: boto3 KMS client. + kms_key_id: KMS key ARN or alias used to encrypt the object. + bucket: S3 bucket containing the encrypted object and instruction files. + key: S3 object key of the encrypted object. + expected_plaintext: Expected plaintext content for verification. + """ + keyring = KmsKeyring(kms_client=kms_client, kms_key_id=kms_key_id) + + # 1. Decrypt using the default instruction file suffix (".instruction"). + # The client will fetch ".instruction" for the encryption metadata. + config = S3EncryptionClientConfig(keyring=keyring) + s3ec = S3EncryptionClient(wrapped_s3_client=s3_client, config=config) + + response = s3ec.get_object(Bucket=bucket, Key=key) + plaintext = response["Body"].read() + assert plaintext == expected_plaintext, "Default suffix: decrypted plaintext does not match" + + # 2. Decrypt while specifying the Instruction File Suffix + # InstructionFileSuffix is a per-request keyword argument on get_object, + # so the same client can use different suffixes per request. + response = s3ec.get_object( + Bucket=bucket, Key=key, InstructionFileSuffix=".custom-suffix-instruction" + ) + plaintext = response["Body"].read() + assert plaintext == expected_plaintext, "Custom suffix: decrypted plaintext does not match" diff --git a/examples/src/kms_keyring_put_get_example.py b/examples/src/kms_keyring_put_get_example.py new file mode 100644 index 00000000..de6ef4e4 --- /dev/null +++ b/examples/src/kms_keyring_put_get_example.py @@ -0,0 +1,95 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +This example demonstrates a basic put/get roundtrip using the S3 Encryption Client +with a KMS Keyring. + +The KMS Keyring uses a symmetric KMS key to generate and decrypt data keys. +The S3 Encryption Client encrypts the object before uploading to S3 and decrypts +it on download, so the data is protected at rest. + +This example: +1. Creates a KMS Keyring with the provided KMS key ID +2. Wraps a boto3 S3 client with the S3 Encryption Client +3. Creates an encryption context bound to the S3 bucket and key +4. Puts an encrypted object to S3 +5. Gets and decrypts the object from S3 +6. Verifies the decrypted plaintext matches the original + +Here is an example KMS Key Policy statement that would validate the +Encryption Context used in this example:: + + Sid: RestrictToEncryptionContextBucket + Effect: Allow + Principal: + AWS: "arn:aws:iam:::role/" + Action: + - kms:GenerateDataKey + - kms:Decrypt + Resource: "*" + Condition: + StringEquals: + "kms:EncryptionContext:aws-s3-bucket": "" +""" + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig, S3EncryptionClientError +from s3_encryption.materials.kms_keyring import KmsKeyring + +EXAMPLE_DATA: bytes = b"Hello, S3 Encryption Client!" + + +def kms_keyring_put_get(s3_client, kms_client, kms_key_id: str, bucket: str, key: str): + """Demonstrate an encrypt/decrypt cycle using a KMS Keyring with S3. + + Args: + s3_client: boto3 S3 client. + kms_client: boto3 KMS client. + kms_key_id: KMS key ARN or alias to use for encryption/decryption. + bucket: S3 bucket name. + key: S3 object key. + """ + # 1. Create a KMS Keyring. + keyring = KmsKeyring(kms_client=kms_client, kms_key_id=kms_key_id) + + # 2. Wrap the S3 client with the S3 Encryption Client. + # The default commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + # which enforces key-committing algorithm suites on both encrypt and decrypt. + config = S3EncryptionClientConfig(keyring=keyring) + s3ec = S3EncryptionClient(wrapped_s3_client=s3_client, config=config) + + # 3. Create an encryption context. + # The encryption context is a set of key-value pairs that are bound to the ciphertext. + # Including the bucket and key ensures the ciphertext is tied to this specific S3 object. + # This will also be visible to KMS when evaluating key policies. + # See the example KMS Key Policy in this module's docstring. + # The encryption context is optional, but strongly recommended. + encryption_context = { + "aws-s3-bucket": bucket, + "aws-s3-key": key, + } + + # 4. Put an encrypted object. + s3ec.put_object(Bucket=bucket, Key=key, Body=EXAMPLE_DATA, EncryptionContext=encryption_context) + + # 5. Get and decrypt the object. + # If you specified an encryption context during encryption, + # you must provide the same encryption context during decryption. + response = s3ec.get_object(Bucket=bucket, Key=key, EncryptionContext=encryption_context) + plaintext = response["Body"].read() + + # 6. Optional Verify the decrypted plaintext matches the original. + assert plaintext == EXAMPLE_DATA, "Decrypted plaintext does not match original data" + + # However, if the Encryption Context is not present at decryption time, then decryption will fail + failed = False + try: + _ = s3ec.get_object( + Bucket=bucket, Key=key, + # Incomplete Encryption Context + EncryptionContext={"aws-s3-bucket": bucket}) + except S3EncryptionClientError as e: + failed = True + assert hasattr(e, "kwargs") + assert e.kwargs.get("msg") is not None + assert e.kwargs.get("msg") == "Provided encryption context does not match information retrieved from S3" + assert failed diff --git a/examples/src/legacy_decrypt_example.py b/examples/src/legacy_decrypt_example.py new file mode 100644 index 00000000..cbccc96b --- /dev/null +++ b/examples/src/legacy_decrypt_example.py @@ -0,0 +1,60 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +This example demonstrates how to decrypt legacy S3 objects that were encrypted +using older versions of the S3 Encryption Client (V1 or V2). + +Legacy objects use the KmsV1 wrapping algorithm and may use unauthenticated +content encryption (AES-CBC). To decrypt these objects, you must: +1. Enable legacy wrapping algorithms on the KMS Keyring +2. Enable legacy unauthenticated modes on the S3 Encryption Client config +3. Use a commitment policy that allows non-key-committing algorithm suites + +This example: +1. Creates a KMS Keyring with legacy wrapping algorithms enabled +2. Configures the S3 Encryption Client with legacy decryption support +3. Decrypts a legacy V1 object from S3 +4. Verifies the decrypted plaintext matches the expected content +""" + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import CommitmentPolicy + + +def decrypt_legacy_object(s3_client, kms_client, kms_key_id: str, bucket: str, key: str): + """Decrypt a legacy S3 object encrypted by an older S3 Encryption Client. + + Args: + s3_client: boto3 S3 client. + kms_client: boto3 KMS client. + kms_key_id: KMS key ARN or alias used to encrypt the object. + bucket: S3 bucket name. + key: S3 object key. + + Returns: + Decrypted plaintext bytes. + """ + # 1. Create a KMS Keyring with legacy wrapping algorithms enabled. + # This allows the keyring to decrypt data keys wrapped using the KmsV1 mode, + # which older S3 Encryption Clients used. + keyring = KmsKeyring( + kms_client=kms_client, + kms_key_id=kms_key_id, + enable_legacy_wrapping_algorithms=True, + ) + + # 2. Configure the S3 Encryption Client for legacy decryption. + # - enable_legacy_unauthenticated_modes: allows decryption of AES-CBC content + # - REQUIRE_ENCRYPT_ALLOW_DECRYPT: new objects are encrypted with key-committing + # algorithm suites, while still allowing decryption of legacy objects + config = S3EncryptionClientConfig( + keyring=keyring, + enable_legacy_unauthenticated_modes=True, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + ) + s3ec = S3EncryptionClient(wrapped_s3_client=s3_client, config=config) + + # 3. Decrypt the legacy object. + response = s3ec.get_object(Bucket=bucket, Key=key) + return response["Body"].read() diff --git a/examples/test/__init__.py b/examples/test/__init__.py new file mode 100644 index 00000000..f94fd12a --- /dev/null +++ b/examples/test/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/examples/test/test_i_delayed_auth_streaming_example.py b/examples/test/test_i_delayed_auth_streaming_example.py new file mode 100644 index 00000000..501c7be0 --- /dev/null +++ b/examples/test/test_i_delayed_auth_streaming_example.py @@ -0,0 +1,27 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Test suite for the delayed auth streaming decrypt example.""" +import boto3 +import pytest + +from ..src.delayed_auth_streaming_example import delayed_auth_streaming_decrypt + +pytestmark = [pytest.mark.examples] + +BUCKET = "s3ec-python-github-test-bucket" +KEY = "examples/delayed-auth-streaming" +KMS_KEY_ID = "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" + + +def test_delayed_auth_streaming_decrypt(): + s3_client = boto3.client("s3", region_name="us-west-2") + kms_client = boto3.client("kms", region_name="us-west-2") + delayed_auth_streaming_decrypt( + s3_client=s3_client, + kms_client=kms_client, + kms_key_id=KMS_KEY_ID, + bucket=BUCKET, + key=KEY, + ) + # Clean up + s3_client.delete_object(Bucket=BUCKET, Key=KEY) diff --git a/examples/test/test_i_instruction_file_example.py b/examples/test/test_i_instruction_file_example.py new file mode 100644 index 00000000..938147f8 --- /dev/null +++ b/examples/test/test_i_instruction_file_example.py @@ -0,0 +1,27 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Test suite for the instruction file example.""" + +import boto3 +import pytest + +from ..src.instruction_file_example import instruction_file_get + +pytestmark = [pytest.mark.examples] + +BUCKET = "s3ec-static-test-objects" +KEY = "static-v3-instruction-file-from-java-v4" +KMS_KEY_ID = "arn:aws:kms:us-west-2:370957321024:key/a3889cd9-99eb-4138-a93a-aea9d52ec2ef" + + +def test_instruction_file_get(): + s3_client = boto3.client("s3", region_name="us-west-2") + kms_client = boto3.client("kms", region_name="us-west-2") + instruction_file_get( + s3_client=s3_client, + kms_client=kms_client, + kms_key_id=KMS_KEY_ID, + bucket=BUCKET, + key=KEY, + expected_plaintext=KEY.encode("utf-8"), + ) diff --git a/examples/test/test_i_kms_keyring_put_get_example.py b/examples/test/test_i_kms_keyring_put_get_example.py new file mode 100644 index 00000000..08759041 --- /dev/null +++ b/examples/test/test_i_kms_keyring_put_get_example.py @@ -0,0 +1,27 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Test suite for the KMS Keyring put/get example.""" +import boto3 +import pytest + +from ..src.kms_keyring_put_get_example import kms_keyring_put_get + +pytestmark = [pytest.mark.examples] + +BUCKET = "s3ec-python-github-test-bucket" +KEY = "examples/kms-keyring-put-get" +KMS_KEY_ID = "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" + + +def test_kms_keyring_put_get(): + s3_client = boto3.client("s3", region_name="us-west-2") + kms_client = boto3.client("kms", region_name="us-west-2") + kms_keyring_put_get( + s3_client=s3_client, + kms_client=kms_client, + kms_key_id=KMS_KEY_ID, + bucket=BUCKET, + key=KEY, + ) + # Clean up + s3_client.delete_object(Bucket=BUCKET, Key=KEY) diff --git a/examples/test/test_i_legacy_decrypt_example.py b/examples/test/test_i_legacy_decrypt_example.py new file mode 100644 index 00000000..93f67d0c --- /dev/null +++ b/examples/test/test_i_legacy_decrypt_example.py @@ -0,0 +1,27 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Test suite for the legacy decrypt example.""" +import boto3 +import pytest + +from ..src.legacy_decrypt_example import decrypt_legacy_object + +pytestmark = [pytest.mark.examples] + +BUCKET = "s3ec-static-test-objects" +KEY = "static-v1-instruction-file-from-java-v1" +KMS_KEY_ID = "arn:aws:kms:us-west-2:370957321024:key/a3889cd9-99eb-4138-a93a-aea9d52ec2ef" + + +def test_decrypt_legacy_object(): + s3_client = boto3.client("s3", region_name="us-west-2") + kms_client = boto3.client("kms", region_name="us-west-2") + plaintext = decrypt_legacy_object( + s3_client=s3_client, + kms_client=kms_client, + kms_key_id=KMS_KEY_ID, + bucket=BUCKET, + key=KEY, + ) + assert plaintext == KEY.encode("utf-8") + # Avoid deleting the static object, it is used in the integration tests diff --git a/pyproject.toml b/pyproject.toml index 93fcbfcf..781e89e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,3 +68,8 @@ source = ["src/s3_encryption"] [tool.coverage.report] show_missing = true + +[tool.pytest.ini_options] +markers = [ + "examples: S3 Encryption Client example tests", +] diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 7ece424c..dd0441d8 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -27,10 +27,16 @@ _CTX_KEY = "key" _CTX_S3_CLIENT = "s3_client" _CTX_INSTRUCTION_FILE_MODE = "instruction_file_mode" +_CTX_INSTRUCTION_FILE_SUFFIX = "instruction_file_suffix" # Attributes to clean up after get_object completes # (s3_client is intentionally excluded — it is not request-scoped) -_GET_OBJECT_CLEANUP_ATTRS = (_CTX_ENCRYPTION_CONTEXT, _CTX_BUCKET, _CTX_KEY) +_GET_OBJECT_CLEANUP_ATTRS = ( + _CTX_ENCRYPTION_CONTEXT, + _CTX_BUCKET, + _CTX_KEY, + _CTX_INSTRUCTION_FILE_SUFFIX, +) @define @@ -47,8 +53,6 @@ class S3EncryptionClientConfig: encrypted with legacy CBC algorithm suites. Defaults to False. cmm: Crypto materials manager. Defaults to a DefaultCryptoMaterialsManager wrapping the provided keyring. - instruction_file_suffix: Suffix appended to the S3 object key when - fetching instruction files. Defaults to ".instruction". enable_delayed_authentication: If True, release plaintext from streams before GCM tag verification. Defaults to False. Has no effect for CBC encrypted ciphertext, which is always streamed as there is no @@ -72,16 +76,6 @@ class S3EncryptionClientConfig: ##% The option to enable legacy unauthenticated modes MUST be set to false by default. enable_legacy_unauthenticated_modes: bool = field(default=False) cmm: AbstractCryptoMaterialsManager = field() - ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file - ##= type=implementation - ##% The S3EC SHOULD support providing a custom Instruction File suffix - ##% on GetObject requests, regardless of whether or not re-encryption is supported. - - ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file - ##= type=implementation - ##% The default Instruction File behavior uses the same S3 object key - ##% as its associated object suffixed with ".instruction". - instruction_file_suffix: str = field(default=".instruction") ##= specification/s3-encryption/client.md#enable-delayed-authentication ##= type=implementation @@ -257,7 +251,7 @@ def on_get_object_after_call(self, parsed, **kwargs): ) decrypted_data = pipeline.decrypt( response, - instruction_suffix=self.config.instruction_file_suffix, + instruction_suffix=getattr(self._context, _CTX_INSTRUCTION_FILE_SUFFIX, ".instruction"), enable_delayed_authentication=self.config.enable_delayed_authentication, encryption_context=encryption_context, bucket=getattr(self._context, _CTX_BUCKET, None), @@ -374,6 +368,8 @@ def get_object(self, **kwargs): Args: **kwargs: Arguments to pass to the S3 client's get_object method. May include EncryptionContext if it was used during encryption. + May include InstructionFileSuffix to override the default + ".instruction" suffix for instruction file lookups. Returns: The response from the S3 client's get_object method with the Body @@ -384,9 +380,20 @@ def get_object(self, **kwargs): """ # Extract EncryptionContext if provided (not a standard S3 parameter) encryption_context = kwargs.pop("EncryptionContext", None) + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=implementation + ##% The S3EC SHOULD support providing a custom Instruction File suffix + ##% on GetObject requests, regardless of whether or not re-encryption is supported. + + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=implementation + ##% The default Instruction File behavior uses the same S3 object key + ##% as its associated object suffixed with ".instruction". + instruction_file_suffix = kwargs.pop("InstructionFileSuffix", ".instruction") # Store encryption context in thread-local storage for the event handler setattr(self._plugin._context, _CTX_ENCRYPTION_CONTEXT, encryption_context) + setattr(self._plugin._context, _CTX_INSTRUCTION_FILE_SUFFIX, instruction_file_suffix) # Store wrapped client in thread-local storage for # the event handler to fetch instruction files setattr(self._plugin._context, _CTX_S3_CLIENT, self.wrapped_s3_client) diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 5561047a..0a107a5f 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -470,8 +470,9 @@ def _decrypt_gcm_streaming( ##% When disabled the S3EC MUST NOT release plaintext from a stream which has not been authenticated. return one_shot_decrypt(streaming_body, decryptor) + @staticmethod def _decrypt_kc_gcm_streaming( - self, dec_materials, metadata, streaming_body, enable_delayed_authentication, content_length + dec_materials, metadata, streaming_body, enable_delayed_authentication, content_length ): """Decrypt content encrypted with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY. diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index 6ca72e8b..a78a6902 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -138,12 +138,13 @@ def test_decrypt_v3_instruction_file_custom_suffix(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig( keyring, - instruction_file_suffix=".custom-suffix-instruction", commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, ) s3ec = S3EncryptionClient(wrapped_client, config) - response = s3ec.get_object(Bucket=bucket, Key=key) + response = s3ec.get_object( + Bucket=bucket, Key=key, InstructionFileSuffix=".custom-suffix-instruction" + ) output = response["Body"].read().decode("utf-8") assert output == "static-v3-instruction-file-from-java-v4" @@ -161,13 +162,14 @@ def test_decrypt_v2_instruction_file_custom_suffix(delayed_auth): config = S3EncryptionClientConfig( keyring, encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, - instruction_file_suffix=".custom-suffix-instruction", commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, enable_delayed_authentication=delayed_auth, ) s3ec = S3EncryptionClient(wrapped_client, config) - response = s3ec.get_object(Bucket=bucket, Key=key) + response = s3ec.get_object( + Bucket=bucket, Key=key, InstructionFileSuffix=".custom-suffix-instruction" + ) output = response["Body"].read().decode("utf-8") assert output == "static-v2-instruction-file-from-java-v4" From 099ab77c913db8b482208875a35bf1a2296f3c42 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Fri, 17 Apr 2026 16:09:54 -0700 Subject: [PATCH 65/81] chore: fill in gaps in testing (#170) * proxy __getattr__ to wrapped client, add several more tests * split coverage between unit and integ tests * add MRK keys, MRK test * validate against non-ASCII chars --- .github/workflows/python-integ.yml | 33 +-- Makefile | 8 +- cdk/lib/cdk-stack.ts | 31 +++ src/s3_encryption/__init__.py | 37 +++ test/integration/test_i_custom_keyring_cmm.py | 239 ++++++++++++++++++ .../test_i_key_commitment_policy.py | 200 +++++++++++++++ test/integration/test_i_mrk_cross_region.py | 123 +++++++++ test/integration/test_i_s3_encryption.py | 129 ++++++++++ .../test_i_s3_encryption_instruction_file.py | 19 ++ test/test_encryption.py | 15 ++ test/test_key_commitment.py | 24 ++ test/test_pipelines.py | 124 +++++++++ test/test_s3_encryption_client_plugin.py | 14 + 13 files changed, 976 insertions(+), 20 deletions(-) create mode 100644 test/integration/test_i_custom_keyring_cmm.py create mode 100644 test/integration/test_i_key_commitment_policy.py create mode 100644 test/integration/test_i_mrk_cross_region.py diff --git a/.github/workflows/python-integ.yml b/.github/workflows/python-integ.yml index d2761518..0f50c8e8 100644 --- a/.github/workflows/python-integ.yml +++ b/.github/workflows/python-integ.yml @@ -48,34 +48,35 @@ jobs: aws-region: us-west-2 - name: Run unit tests - run: make test-unit + run: | + uv run pytest test/ --ignore=test/integration/ --verbose \ + --cov=src/s3_encryption --cov-report=term-missing --cov-report=html:coverage-unit \ + --cov-fail-under=89 - name: Run integration tests - run: make test-integration + run: | + uv run pytest test/integration/ --verbose \ + --cov=src/s3_encryption --cov-report=term-missing --cov-report=html:coverage-integ \ + --cov-fail-under=83 env: CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} + CI_MRK_KEY_ID_PRIMARY: ${{ vars.CI_MRK_KEY_ID_PRIMARY }} + CI_MRK_KEY_ID_REPLICA: ${{ vars.CI_MRK_KEY_ID_REPLICA }} - name: Run examples run: make test-examples - name: Generate coverage HTML report if: always() - run: uv run coverage html -d coverage-report + uses: actions/upload-artifact@v7 + with: + name: coverage-unit + path: coverage-unit/ - - name: Upload coverage report + - name: Upload integration test coverage report if: always() uses: actions/upload-artifact@v7 with: - name: coverage-report - path: coverage-report/ - - - name: Check coverage threshold - run: | - THRESHOLD=93 - ACTUAL=$(uv run coverage report --format=total) - echo "Coverage: ${ACTUAL}% (threshold: ${THRESHOLD}%)" - if [ "$ACTUAL" -gt "$THRESHOLD" ]; then - echo "::warning::Coverage is ${ACTUAL}%, consider updating --fail-under to ${ACTUAL} in python-integ.yml" - fi - uv run coverage report --fail-under=$THRESHOLD + name: coverage-integ + path: coverage-integ/ diff --git a/Makefile b/Makefile index 5960117a..caf05737 100644 --- a/Makefile +++ b/Makefile @@ -23,13 +23,13 @@ format: # Run all tests with combined coverage test: test-unit test-integration test-examples -# Run unit tests (creates .coverage report) +# Run unit tests with coverage test-unit: - uv run pytest test/ --ignore=test/integration/ --verbose --cov=src/s3_encryption --cov-report=term-missing + uv run pytest test/ --ignore=test/integration/ --verbose --cov=src/s3_encryption --cov-report=term-missing --cov-fail-under=89 -# Run integration tests (appends to .coverage report from test-unit) +# Run integration tests with separate coverage test-integration: - uv run pytest test/integration/ --verbose --cov=src/s3_encryption --cov-append --cov-report=term-missing + uv run pytest test/integration/ --verbose --cov=src/s3_encryption --cov-report=term-missing --cov-fail-under=83 test-examples: uv run pytest examples/test/ -v diff --git a/cdk/lib/cdk-stack.ts b/cdk/lib/cdk-stack.ts index b5a28084..329e16ca 100644 --- a/cdk/lib/cdk-stack.ts +++ b/cdk/lib/cdk-stack.ts @@ -65,6 +65,33 @@ export class S3ECPythonGithub extends cdk.Stack { } ) + // Multi-Region Key (MRK) for cross-region testing. + // The primary key is created here in the stack's region (us-west-2). + // A replica MUST be created manually in us-east-1 via the AWS Console + // or a separate CDK stack, since CDK cannot create cross-region replicas + // within a single stack. + const S3ECMRKPrimaryKey = new Key( + this, + "S3ECMRKPrimaryKey", + { + enableKeyRotation: true, + description: "Multi-Region primary key for S3EC cross-region testing", + // multiRegion is not a direct CDK L2 prop; use cfnOptions override + } + ); + // Override to enable multi-region on the underlying CloudFormation resource + const cfnMrkKey = S3ECMRKPrimaryKey.node.defaultChild as cdk.aws_kms.CfnKey; + cfnMrkKey.addPropertyOverride("MultiRegion", true); + + const S3ECMRKPrimaryKeyAlias = new Alias( + this, + "S3ECMRKPrimaryKeyAlias", + { + aliasName: "alias/S3EC-Python-MRK-Primary", + targetKey: S3ECMRKPrimaryKey, + } + ); + // S3 buckets const AccessConfiguration: BlockPublicAccessOptions = { blockPublicAcls: false, @@ -162,6 +189,10 @@ export class S3ECPythonGithub extends cdk.Stack { resources: [ S3ECGithubKMSKey.keyArn, S3ECTestServerKMSKey.keyArn, // Add access to the test-server KMS key + S3ECMRKPrimaryKey.keyArn, // MRK primary key + // MRK replica in us-east-1 — ARN must use wildcard account + // since the replica shares the same key ID but different region + `arn:aws:kms:us-east-1:${this.account}:key/${S3ECMRKPrimaryKey.keyId}`, ] }) ] diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index dd0441d8..a9d5a3a1 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -294,6 +294,32 @@ def process_instruction_file(self, parsed): parsed["Body"] = streaming_body +def _validate_encryption_context(encryption_context): + """Validate that all encryption context keys and values are US-ASCII. + + S3 applies double-encoding to non-ASCII metadata values that SDKs do not + automatically decode, which causes decryption to fail because the stored + encryption context won't match the original. + + Raises: + S3EncryptionClientError: If any key or value contains non-ASCII characters. + """ + if encryption_context is None: + return + if not isinstance(encryption_context, dict): + raise S3EncryptionClientError("EncryptionContext must be a dictionary") + for k, v in encryption_context.items(): + if not isinstance(k, str) or not isinstance(v, str): + raise S3EncryptionClientError("EncryptionContext keys and values must be strings") + if not k.isascii() or not v.isascii(): + raise S3EncryptionClientError( + f"EncryptionContext keys and values must contain only US-ASCII characters. " + f"Non-ASCII characters in S3 metadata are encoded by the server " + f"and will cause decryption to fail. " + f"First offending entry: {repr(k)}: {repr(v)}" + ) + + @define class S3EncryptionClient: """Client for encrypting and decrypting S3 objects. @@ -322,6 +348,15 @@ def __attrs_post_init__(self): event_system.register("before-call.s3.PutObject", self._plugin.on_put_object_before_call) event_system.register("after-call.s3.GetObject", self._plugin.on_get_object_after_call) + def __getattr__(self, name): + """Proxy unrecognized attributes to the wrapped S3 client. + + This allows the S3EncryptionClient to be used like a regular boto3 S3 + client for operations it doesn't intercept (e.g. copy_object, + list_objects_v2, etc.). + """ + return getattr(self.wrapped_s3_client, name) + def put_object(self, **kwargs): """Encrypt and upload an object to S3. @@ -343,6 +378,7 @@ def put_object(self, **kwargs): """ # Extract EncryptionContext if provided (not a standard S3 parameter) encryption_context = kwargs.pop("EncryptionContext", None) + _validate_encryption_context(encryption_context) # Store encryption context in thread-local storage for the event handler self._plugin._context.encryption_context = encryption_context @@ -380,6 +416,7 @@ def get_object(self, **kwargs): """ # Extract EncryptionContext if provided (not a standard S3 parameter) encryption_context = kwargs.pop("EncryptionContext", None) + _validate_encryption_context(encryption_context) ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file ##= type=implementation ##% The S3EC SHOULD support providing a custom Instruction File suffix diff --git a/test/integration/test_i_custom_keyring_cmm.py b/test/integration/test_i_custom_keyring_cmm.py new file mode 100644 index 00000000..45cd3441 --- /dev/null +++ b/test/integration/test_i_custom_keyring_cmm.py @@ -0,0 +1,239 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Integration tests for custom keyring and custom CMM. + +These tests verify that user-implemented AbstractKeyring and +AbstractCryptoMaterialsManager subclasses work end-to-end through +S3EncryptionClient.put_object / get_object. + +WARNING: The custom classes below are test-only stubs that duplicate the +built-in KmsKeyring and DefaultCryptoMaterialsManager logic. They exist +solely to prove the extension points work. Do NOT use them in production. +""" + +import os +from datetime import datetime + +import boto3 + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.materials.crypto_materials_manager import AbstractCryptoMaterialsManager +from s3_encryption.materials.encrypted_data_key import EncryptedDataKey +from s3_encryption.materials.keyring import S3Keyring +from s3_encryption.materials.materials import ( + AlgorithmSuite, + CommitmentPolicy, + DecryptionMaterials, + EncryptionMaterials, +) + +bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") +region = os.environ.get("CI_AWS_REGION", "us-west-2") +kms_key_id = os.environ.get( + "CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" +) + +KMS_CONTEXT_DEFAULT_KEY = "aws:x-amz-cek-alg" + + +# --------------------------------------------------------------------------- +# Custom keyring — test-only, do NOT use in production code. +# Duplicates KmsKeyring logic to prove the AbstractKeyring extension point. +# --------------------------------------------------------------------------- + + +class CustomTestKmsKeyring(S3Keyring): + """Test-only KMS keyring. Do NOT use in production.""" + + def __init__(self, kms_client, kms_key_id): + self.kms_client = kms_client + self.kms_key_id = kms_key_id + + def on_encrypt(self, enc_materials): + enc_materials = super().on_encrypt(enc_materials) + encryption_context = enc_materials.encryption_context + + if ( + enc_materials.encryption_algorithm + == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ): + encryption_context[KMS_CONTEXT_DEFAULT_KEY] = str( + enc_materials.encryption_algorithm.suite_id + ) + else: + encryption_context[KMS_CONTEXT_DEFAULT_KEY] = ( + enc_materials.encryption_algorithm.cipher_name + ) + + response = self.kms_client.generate_data_key( + KeyId=self.kms_key_id, KeySpec="AES_256", EncryptionContext=encryption_context + ) + enc_materials.encrypted_data_key = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms+context", + encrypted_data_key=response["CiphertextBlob"], + ) + enc_materials.plaintext_data_key = response["Plaintext"] + return enc_materials + + def on_decrypt(self, dec_materials, encrypted_data_keys=None): + dec_materials = super().on_decrypt(dec_materials, encrypted_data_keys) + edks = ( + encrypted_data_keys + if encrypted_data_keys is not None + else dec_materials.encrypted_data_keys + ) + edk = edks[0] + + if edk.key_provider_info == "kms+context": + ec_from_request = dec_materials.encryption_context_from_request + ec_stored = dec_materials.encryption_context_stored + + if KMS_CONTEXT_DEFAULT_KEY in ec_from_request: + raise S3EncryptionClientError(f"{KMS_CONTEXT_DEFAULT_KEY} is a reserved key") + + ec_stored_copy = ec_stored.copy() + ec_stored_copy.pop("kms_cmk_id", None) + ec_stored_copy.pop(KMS_CONTEXT_DEFAULT_KEY, None) + + if ec_stored_copy != ec_from_request: + raise S3EncryptionClientError("Provided encryption context does not match") + elif edk.key_provider_info != "kms": + raise S3EncryptionClientError( + f"{edk.key_provider_info} is not a valid key wrapping algorithm!" + ) + + response = self.kms_client.decrypt( + KeyId=self.kms_key_id, + CiphertextBlob=edk.encrypted_data_key, + EncryptionContext=dec_materials.encryption_context_stored, + ) + dec_materials.plaintext_data_key = response["Plaintext"] + return dec_materials + + +# --------------------------------------------------------------------------- +# Custom CMM — test-only, do NOT use in production code. +# Duplicates DefaultCryptoMaterialsManager logic to prove the CMM extension point. +# --------------------------------------------------------------------------- + + +class CustomTestCMM(AbstractCryptoMaterialsManager): + """Test-only CMM. Do NOT use in production.""" + + def __init__(self, keyring): + self.keyring = keyring + + def get_encryption_materials(self, enc_mats_request): + if isinstance(enc_mats_request, dict): + materials = EncryptionMaterials( + encryption_context=enc_mats_request.get("encryption_context", {}) + ) + else: + materials = enc_mats_request + return self.keyring.on_encrypt(materials) + + def decrypt_materials(self, dec_mats_request): + if isinstance(dec_mats_request, dict): + materials = DecryptionMaterials.from_dict(dec_mats_request) + else: + materials = dec_mats_request + return self.keyring.on_decrypt(materials, materials.encrypted_data_keys) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _unique_key(prefix): + return prefix + datetime.now().strftime("%Y-%m-%d-%H:%M:%S-%f") + + +# --------------------------------------------------------------------------- +# Integration tests +# --------------------------------------------------------------------------- + + +class TestCustomKeyring: + """Verify a user-implemented AbstractKeyring subclass works end-to-end.""" + + def test_roundtrip_with_custom_keyring(self): + """Custom keyring MUST encrypt and decrypt successfully.""" + kms_client = boto3.client("kms", region_name=region) + keyring = CustomTestKmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring=keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + key = _unique_key("custom-keyring-rt-") + data = b"custom keyring round trip test" + + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == data + + def test_roundtrip_with_custom_keyring_aes_gcm(self): + """Custom keyring MUST work with non-committing AES-GCM suite.""" + kms_client = boto3.client("kms", region_name=region) + keyring = CustomTestKmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig( + keyring=keyring, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) + s3ec = S3EncryptionClient(wrapped_client, config) + + key = _unique_key("custom-keyring-gcm-rt-") + data = b"custom keyring AES-GCM round trip" + + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == data + + +class TestCustomCMM: + """Verify a user-implemented AbstractCryptoMaterialsManager subclass works end-to-end.""" + + def test_roundtrip_with_custom_cmm(self): + """Custom CMM MUST encrypt and decrypt successfully.""" + from s3_encryption.materials.kms_keyring import KmsKeyring + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + custom_cmm = CustomTestCMM(keyring) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring=keyring, cmm=custom_cmm) + s3ec = S3EncryptionClient(wrapped_client, config) + + key = _unique_key("custom-cmm-rt-") + data = b"custom CMM round trip test" + + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == data + + def test_roundtrip_with_custom_cmm_aes_gcm(self): + """Custom CMM MUST work with non-committing AES-GCM suite.""" + from s3_encryption.materials.kms_keyring import KmsKeyring + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + custom_cmm = CustomTestCMM(keyring) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig( + keyring=keyring, + cmm=custom_cmm, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) + s3ec = S3EncryptionClient(wrapped_client, config) + + key = _unique_key("custom-cmm-gcm-rt-") + data = b"custom CMM AES-GCM round trip" + + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == data diff --git a/test/integration/test_i_key_commitment_policy.py b/test/integration/test_i_key_commitment_policy.py new file mode 100644 index 00000000..ba334a87 --- /dev/null +++ b/test/integration/test_i_key_commitment_policy.py @@ -0,0 +1,200 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Integration tests for key commitment policy enforcement through the front door. + +These tests verify that commitment policy behavior works end-to-end through +S3EncryptionClient.put_object / get_object, not just at the pipeline level. + +Objects are encrypted with one policy and decrypted with another to verify +cross-policy compatibility. +""" + +import os +from datetime import datetime + +import boto3 +import pytest + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy + +bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") +region = os.environ.get("CI_AWS_REGION", "us-west-2") +kms_key_id = os.environ.get( + "CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" +) + + +def _make_client(algorithm_suite, commitment_policy): + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig( + keyring, + encryption_algorithm=algorithm_suite, + commitment_policy=commitment_policy, + ) + return S3EncryptionClient(wrapped_client, config) + + +def _unique_key(prefix): + return prefix + datetime.now().strftime("%Y-%m-%d-%H:%M:%S-%f") + + +# --------------------------------------------------------------------------- +# Non-committing (V2 GCM) objects decrypted under various policies +# --------------------------------------------------------------------------- + + +class TestNonCommittingObjectDecryptPolicies: + """Verify V2 (non-committing) objects can be decrypted under ALLOW policies + and rejected under REQUIRE_REQUIRE. + """ + + PLAINTEXT = b"non-committing policy integration test" + + @pytest.fixture(autouse=True, scope="class") + def _encrypt_v2_object(self, request): + """Encrypt a single V2 object to be shared across all tests in this class.""" + key = _unique_key("kc-v2-policy-") + writer = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) + writer.put_object(Bucket=bucket, Key=key, Body=self.PLAINTEXT) + request.cls.s3_key = key + + def test_forbid_encrypt_allow_decrypt_decrypts_non_committing(self): + """FORBID_ENCRYPT_ALLOW_DECRYPT MUST decrypt non-committing objects.""" + reader = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) + response = reader.get_object(Bucket=bucket, Key=self.s3_key) + assert response["Body"].read() == self.PLAINTEXT + + def test_require_encrypt_allow_decrypt_decrypts_non_committing(self): + """REQUIRE_ENCRYPT_ALLOW_DECRYPT MUST decrypt non-committing objects.""" + reader = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + ) + response = reader.get_object(Bucket=bucket, Key=self.s3_key) + assert response["Body"].read() == self.PLAINTEXT + + def test_require_require_rejects_non_committing(self): + """REQUIRE_ENCRYPT_REQUIRE_DECRYPT MUST reject non-committing objects.""" + reader = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + with pytest.raises(S3EncryptionClientError, match="cannot decrypt non-key-committing"): + reader.get_object(Bucket=bucket, Key=self.s3_key) + + +# --------------------------------------------------------------------------- +# Committing (V3 KC-GCM) objects decrypted under various policies +# --------------------------------------------------------------------------- + +# Writer policies that produce committing (V3) objects +COMMITTING_WRITER_POLICIES = [ + pytest.param( + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + id="writer=REQUIRE_REQUIRE", + ), + pytest.param( + CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + id="writer=REQUIRE_ALLOW", + ), +] + + +@pytest.mark.parametrize("writer_policy", COMMITTING_WRITER_POLICIES) +class TestCommittingObjectDecryptPolicies: + """Verify V3 (committing) objects can be decrypted under all three policies, + regardless of which REQUIRE_ENCRYPT_* policy was used to write them. + """ + + PLAINTEXT = b"committing policy integration test" + + @pytest.fixture(autouse=True) + def _encrypt_v3_object(self, writer_policy): + """Encrypt a V3 object with the parametrized writer policy.""" + key = _unique_key("kc-v3-policy-") + writer = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + writer_policy, + ) + writer.put_object(Bucket=bucket, Key=key, Body=self.PLAINTEXT) + self.s3_key = key + + def test_require_require_decrypts_committing(self): + """REQUIRE_ENCRYPT_REQUIRE_DECRYPT MUST decrypt committing objects.""" + reader = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + response = reader.get_object(Bucket=bucket, Key=self.s3_key) + assert response["Body"].read() == self.PLAINTEXT + + def test_require_encrypt_allow_decrypt_decrypts_committing(self): + """REQUIRE_ENCRYPT_ALLOW_DECRYPT MUST decrypt committing objects.""" + reader = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + ) + response = reader.get_object(Bucket=bucket, Key=self.s3_key) + assert response["Body"].read() == self.PLAINTEXT + + def test_forbid_encrypt_allow_decrypt_decrypts_committing(self): + """FORBID_ENCRYPT_ALLOW_DECRYPT MUST decrypt committing objects.""" + reader = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) + response = reader.get_object(Bucket=bucket, Key=self.s3_key) + assert response["Body"].read() == self.PLAINTEXT + + +# --------------------------------------------------------------------------- +# Encrypt-side config rejection (no S3 needed, but verifies front-door behavior) +# --------------------------------------------------------------------------- + + +class TestEncryptPolicyRejection: + """Verify that incompatible algorithm + policy combos are rejected at config time.""" + + def test_require_encrypt_allow_decrypt_rejects_non_committing(self): + """REQUIRE_ENCRYPT_ALLOW_DECRYPT MUST reject non-committing algorithm at config time.""" + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + with pytest.raises(S3EncryptionClientError): + S3EncryptionClientConfig( + keyring, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + ) + + def test_require_encrypt_require_decrypt_rejects_non_committing(self): + """REQUIRE_ENCRYPT_REQUIRE_DECRYPT MUST reject non-committing algorithm at config time.""" + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + with pytest.raises(S3EncryptionClientError): + S3EncryptionClientConfig( + keyring, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + + def test_forbid_encrypt_allow_decrypt_rejects_committing(self): + """FORBID_ENCRYPT_ALLOW_DECRYPT MUST reject committing algorithm at config time.""" + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + with pytest.raises(S3EncryptionClientError): + S3EncryptionClientConfig( + keyring, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) diff --git a/test/integration/test_i_mrk_cross_region.py b/test/integration/test_i_mrk_cross_region.py new file mode 100644 index 00000000..bb9bd538 --- /dev/null +++ b/test/integration/test_i_mrk_cross_region.py @@ -0,0 +1,123 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Integration tests for Multi-Region Key (MRK) cross-region encrypt/decrypt. + +These tests verify that data encrypted with a KMS MRK primary key in one region +can be decrypted using the MRK replica in another region, and vice versa. + +Prerequisites: + - A KMS MRK primary key in us-west-2 (created by CDK stack) + - A KMS MRK replica of the same key in us-east-1 (created manually after CDK deploy) + - Both keys share the same key ID (mrk-...) but have different region ARNs + +Environment variables: + CI_MRK_KEY_ID_PRIMARY: ARN or alias of the MRK primary in us-west-2 + CI_MRK_KEY_ID_REPLICA: ARN of the MRK replica in us-east-1 + CI_S3_BUCKET: S3 bucket for test objects (us-west-2) +""" + +import os +from datetime import datetime + +import boto3 +import pytest + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.materials.kms_keyring import KmsKeyring + +bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") +primary_region = os.environ.get("CI_AWS_REGION", "us-west-2") +replica_region = "us-east-1" + +mrk_primary = os.environ.get( + "CI_MRK_KEY_ID_PRIMARY", + "arn:aws:kms:us-west-2:370957321024:key/mrk-cea4cf67c6a046ba829f61f69db5c191", +) +mrk_replica = os.environ.get( + "CI_MRK_KEY_ID_REPLICA", + "arn:aws:kms:us-east-1:370957321024:key/mrk-cea4cf67c6a046ba829f61f69db5c191", +) + + +def _make_client(kms_region, kms_key_id): + """Create an S3EncryptionClient using a KMS client in the given region.""" + kms_client = boto3.client("kms", region_name=kms_region) + keyring = KmsKeyring(kms_client, kms_key_id) + # Always use a primary region S3 client + wrapped_client = boto3.client("s3", region_name=primary_region) + config = S3EncryptionClientConfig(keyring=keyring) + return S3EncryptionClient(wrapped_client, config) + + +def _unique_key(prefix): + return prefix + datetime.now().strftime("%Y-%m-%d-%H:%M:%S-%f") + + +class TestMRKCrossRegion: + """Verify MRK encrypt/decrypt works across regions.""" + + def test_encrypt_primary_decrypt_replica(self): + """Data encrypted with MRK primary MUST decrypt with MRK replica.""" + key = _unique_key("mrk-primary-to-replica-") + data = b"MRK cross-region: primary -> replica" + + writer = _make_client(primary_region, mrk_primary) + writer.put_object(Bucket=bucket, Key=key, Body=data) + + reader = _make_client(replica_region, mrk_replica) + response = reader.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == data + + def test_encrypt_replica_decrypt_primary(self): + """Data encrypted with MRK replica MUST decrypt with MRK primary.""" + key = _unique_key("mrk-replica-to-primary-") + data = b"MRK cross-region: replica -> primary" + + writer = _make_client(replica_region, mrk_replica) + writer.put_object(Bucket=bucket, Key=key, Body=data) + + reader = _make_client(primary_region, mrk_primary) + response = reader.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == data + + def test_encrypt_and_decrypt_same_region_primary(self): + """MRK primary round-trip in the same region MUST work.""" + key = _unique_key("mrk-same-region-primary-") + data = b"MRK same-region primary round trip" + + s3ec = _make_client(primary_region, mrk_primary) + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == data + + def test_encrypt_and_decrypt_same_region_replica(self): + """MRK replica round-trip in the same region MUST work.""" + key = _unique_key("mrk-same-region-replica-") + data = b"MRK same-region replica round trip" + + s3ec = _make_client(replica_region, mrk_replica) + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == data + + +class TestMRKNonReplicatedRegionFails: + """Verify that using an MRK in a region where it hasn't been replicated fails.""" + + def test_decrypt_with_wrong_region_kms_client_fails(self): + """Decrypting with a KMS client pointed at a non-replicated region MUST fail.""" + key = _unique_key("mrk-wrong-region-") + data = b"MRK wrong region test" + + # Encrypt with primary + writer = _make_client(primary_region, mrk_primary) + writer.put_object(Bucket=bucket, Key=key, Body=data) + + # Try to decrypt using a KMS client in a region where the MRK doesn't exist. + # Use eu-west-1 as a region that almost certainly has no replica. + non_replicated_region = "eu-west-1" + reader = _make_client(non_replicated_region, mrk_primary) + + with pytest.raises(S3EncryptionClientError): + reader.get_object(Bucket=bucket, Key=key) diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index 36f826bd..4074c6ed 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -138,6 +138,21 @@ def test_binary_data_roundtrip(algorithm_suite, commitment_policy): assert output == data +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_bytesio_body_roundtrip(algorithm_suite, commitment_policy): + """Test that a BytesIO body is encrypted and decrypted correctly.""" + from io import BytesIO + + key = _unique_key("bytesio-body-rt-") + data = b"BytesIO round trip test data" + + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.put_object(Bucket=bucket, Key=key, Body=BytesIO(data)) + response = s3ec.get_object(Bucket=bucket, Key=key) + output = response["Body"].read() + assert output == data + + @pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) def test_invalid_body_types(algorithm_suite, commitment_policy): """Test that put_object raises an exception when given invalid body types.""" @@ -275,3 +290,117 @@ def test_delayed_authentication_mode(enable_delayed_auth): s3ec.put_object(Bucket=bucket, Key=key, Body=data) response = s3ec.get_object(Bucket=bucket, Key=key) assert response["Body"].read() == data + + +def test_inaccessible_kms_key_raises_access_denied(): + """put_object with a KMS key we lack permission for MUST surface AccessDeniedException.""" + from botocore.exceptions import ClientError + + fake_key_arn = "arn:aws:kms:us-west-2:123456789012:key/00000000-0000-0000-0000-000000000000" + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, fake_key_arn) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring=keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + key = _unique_key("access-denied-") + + with pytest.raises(S3EncryptionClientError, match="Failed to encrypt object") as exc_info: + s3ec.put_object(Bucket=bucket, Key=key, Body=b"should fail") + + # Unwrap and verify the root cause is AccessDeniedException + cause = exc_info.value.__cause__ + assert isinstance(cause, ClientError) + assert cause.response["Error"]["Code"] == "AccessDeniedException" + + +def test_get_nonexistent_object_raises_no_such_key(): + """get_object for a key that doesn't exist MUST surface NoSuchKey.""" + from botocore.exceptions import ClientError + + s3ec = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + + with pytest.raises(S3EncryptionClientError, match="NoSuchKey") as exc_info: + s3ec.get_object(Bucket=bucket, Key="this-key-definitely-does-not-exist") + + cause = exc_info.value.__cause__ + assert isinstance(cause, ClientError) + assert cause.response["Error"]["Code"] == "NoSuchKey" + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_s3_passthrough_options_preserved(algorithm_suite, commitment_policy): + """S3 options unrelated to encryption (e.g. StorageClass, ContentType) MUST be applied.""" + key = _unique_key("passthrough-opts-") + data = b'{"message": "hello"}' + + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.put_object( + Bucket=bucket, + Key=key, + Body=data, + StorageClass="STANDARD_IA", + ContentType="application/json", + ContentDisposition="attachment; filename=test.json", + ) + + # Read back with head_object via the S3EC instance to verify the options were applied + head = s3ec.head_object(Bucket=bucket, Key=key) + assert head["StorageClass"] == "STANDARD_IA" + assert head["ContentType"] == "application/json" + assert head["ContentDisposition"] == "attachment; filename=test.json" + + # Also verify the data round-trips correctly + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == data + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_copy_object_then_decrypt(algorithm_suite, commitment_policy): + """An encrypted object copied via CopyObject MUST still decrypt correctly.""" + src_key = _unique_key("copy-src-") + dst_key = _unique_key("copy-dst-") + data = b"copy object round trip test" + + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.put_object(Bucket=bucket, Key=src_key, Body=data) + + # Copy using the S3EC instance (copy_object proxies to the wrapped S3 client) + s3ec.copy_object( + Bucket=bucket, + Key=dst_key, + CopySource={"Bucket": bucket, "Key": src_key}, + MetadataDirective="COPY", + ) + + # Decrypt the copied object + response = s3ec.get_object(Bucket=bucket, Key=dst_key) + assert response["Body"].read() == data + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_non_ascii_encryption_context_rejected(algorithm_suite, commitment_policy): + """Non-US-ASCII characters in EncryptionContext MUST be rejected. + + S3 applies an esoteric double-encoding to non-ASCII metadata values that + most SDKs do not automatically decode. This causes decryption to fail + because the stored encryption context won't match the original. Currently + boto3 rejects non-ASCII header values before the request is sent. + """ + key = _unique_key("non-ascii-ec-") + non_ascii_contexts = [ + {"department": "ingeniería"}, # Latin accented + {"部門": "engineering"}, # CJK key + {"project": "проект"}, # Cyrillic value + {"emoji": "test 🔑"}, # Emoji + {"long😮‍💨": "𐀂"}, # Long Sigh/Psi + ] + + s3ec = _make_client(algorithm_suite, commitment_policy) + + for ec in non_ascii_contexts: + with pytest.raises(S3EncryptionClientError, match="US-ASCII"): + s3ec.put_object(Bucket=bucket, Key=key, Body=b"test", EncryptionContext=ec) diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index a78a6902..b2d2b5f4 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -129,6 +129,25 @@ def test_decrypt_invalid_instruction_file(): print(f"Error message: {exc_info.value}") +def test_decrypt_instruction_file_wrong_suffix_raises(): + """Decryption MUST fail when the instruction file suffix doesn't match the actual S3 object.""" + from s3_encryption.exceptions import S3EncryptionClientError + + key = TEST_OBJECTS["v3_instruction_file"] + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig( + keyring, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + s3ec = S3EncryptionClient(wrapped_client, config) + + with pytest.raises(S3EncryptionClientError, match="Instruction file body is empty"): + s3ec.get_object(Bucket=bucket, Key=key, InstructionFileSuffix=".wrong-suffix") + + def test_decrypt_v3_instruction_file_custom_suffix(): """Test decrypting V3 object with a custom instruction file suffix.""" key = TEST_OBJECTS["v3_instruction_file"] diff --git a/test/test_encryption.py b/test/test_encryption.py index a384afbd..ede9262c 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -135,6 +135,21 @@ def test_message_id_included_in_metadata_kc(self): _, meta = pipeline.encrypt(b"test") assert "x-amz-i" in meta + def test_bytesio_body_encrypts_successfully(self): + """Encryption MUST work when the body is a BytesIO object.""" + cmm, key = _mock_cmm() + pipeline = PutEncryptedObjectPipeline( + cmm, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) + plaintext = b"BytesIO body test data" + + # The plugin reads BytesIO via .read(), so the pipeline receives bytes. + # Verify the pipeline encrypts bytes from a BytesIO source correctly. + ciphertext, meta = pipeline.encrypt(plaintext) + assert ciphertext != plaintext + assert len(ciphertext) > 0 + assert "x-amz-i" in meta # V3 message ID present + # --------------------------------------------------------------------------- # ALG_AES_256_GCM_IV12_TAG16_NO_KDF diff --git a/test/test_key_commitment.py b/test/test_key_commitment.py index a0be12ce..673bf5da 100644 --- a/test/test_key_commitment.py +++ b/test/test_key_commitment.py @@ -163,3 +163,27 @@ def test_require_require_allows_committing_decrypt(self): ) result = pipeline.decrypt(response, ".instruction", enable_delayed_authentication=False) assert result.read() == plaintext + + def test_require_encrypt_allow_decrypt_allows_committing_decrypt(self): + """REQUIRE_ENCRYPT_ALLOW_DECRYPT MUST allow decryption with committing suites.""" + key = os.urandom(32) + response, dec_mats, plaintext = _v3_kc_gcm_response(key) + + pipeline = _make_pipeline( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + keyring_return=dec_mats, + ) + result = pipeline.decrypt(response, ".instruction", enable_delayed_authentication=False) + assert result.read() == plaintext + + def test_forbid_encrypt_allow_decrypt_allows_committing_decrypt(self): + """FORBID_ENCRYPT_ALLOW_DECRYPT MUST allow decryption with committing suites.""" + key = os.urandom(32) + response, dec_mats, plaintext = _v3_kc_gcm_response(key) + + pipeline = _make_pipeline( + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + keyring_return=dec_mats, + ) + result = pipeline.decrypt(response, ".instruction", enable_delayed_authentication=False) + assert result.read() == plaintext diff --git a/test/test_pipelines.py b/test/test_pipelines.py index e3d34e35..a66880e4 100644 --- a/test/test_pipelines.py +++ b/test/test_pipelines.py @@ -310,3 +310,127 @@ def test_decrypt_v3_unsupported_wrap_alg(self): S3EncryptionClientError, match="AES/GCM is not a valid key wrapping algorithm" ): pipeline.decrypt(mock_response, ".instruction", enable_delayed_authentication=False) + + def test_decrypt_instruction_file_no_s3_client_raises(self): + """Instruction file fetch MUST fail when no s3_client is available.""" + # Object metadata has no EDK — triggers instruction file path + object_metadata = {} + + mock_keyring = Mock(spec=S3Keyring) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + s3_client=None, # No s3_client + ) + + mock_response = { + "Body": BytesIO(b"encrypted-data"), + "Metadata": object_metadata, + } + + with pytest.raises(S3EncryptionClientError, match="s3_client required"): + pipeline.decrypt( + mock_response, + instruction_suffix=".instruction", + enable_delayed_authentication=False, + bucket="test-bucket", + key="test-key", + ) + + def test_decrypt_instruction_file_missing_bucket_key_raises(self): + """Instruction file fetch MUST fail when Bucket or Key is missing.""" + object_metadata = {} + + mock_s3_client = Mock() + mock_keyring = Mock(spec=S3Keyring) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + s3_client=mock_s3_client, + ) + + mock_response = { + "Body": BytesIO(b"encrypted-data"), + "Metadata": object_metadata, + } + + with pytest.raises(S3EncryptionClientError, match="Bucket and key required"): + pipeline.decrypt( + mock_response, + instruction_suffix=".instruction", + enable_delayed_authentication=False, + bucket=None, + key=None, + ) + + def test_decrypt_instruction_file_s3_not_found_raises(self): + """Instruction file fetch MUST fail when the file doesn't exist in S3.""" + from botocore.exceptions import ClientError + + object_metadata = {} + + mock_s3_client = Mock() + mock_s3_client.get_object.side_effect = ClientError( + {"Error": {"Code": "NoSuchKey", "Message": "The specified key does not exist."}}, + "GetObject", + ) + # The fetch_instruction_file function checks for _s3ec_plugin_context + mock_s3_client._s3ec_plugin_context = Mock() + + mock_keyring = Mock(spec=S3Keyring) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + s3_client=mock_s3_client, + ) + + mock_response = { + "Body": BytesIO(b"encrypted-data"), + "Metadata": object_metadata, + } + + with pytest.raises(S3EncryptionClientError, match="Instruction File"): + pipeline.decrypt( + mock_response, + instruction_suffix=".instruction", + enable_delayed_authentication=False, + bucket="test-bucket", + key="test-key", + ) + + def test_decrypt_instruction_file_empty_metadata_raises(self): + """Instruction file with no valid metadata MUST raise an error.""" + object_metadata = {} + + mock_s3_client = Mock() + # Instruction file returns empty metadata (empty body parsed to nothing) + mock_s3_client.get_object.return_value = { + "Body": BytesIO(b""), + "Metadata": {}, + } + mock_s3_client._s3ec_plugin_context = Mock() + + mock_keyring = Mock(spec=S3Keyring) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + s3_client=mock_s3_client, + ) + + mock_response = { + "Body": BytesIO(b"encrypted-data"), + "Metadata": object_metadata, + } + + with pytest.raises(S3EncryptionClientError, match="empty metadata"): + pipeline.decrypt( + mock_response, + instruction_suffix=".instruction", + enable_delayed_authentication=False, + bucket="test-bucket", + key="test-key", + ) diff --git a/test/test_s3_encryption_client_plugin.py b/test/test_s3_encryption_client_plugin.py index cbc8cd80..1c930a3a 100644 --- a/test/test_s3_encryption_client_plugin.py +++ b/test/test_s3_encryption_client_plugin.py @@ -154,3 +154,17 @@ def test_missing_content_length_raises_error(self): with pytest.raises(S3EncryptionClientError, match="missing ContentLength.*Key: my-object"): plugin.on_get_object_after_call(parsed) + + def test_put_object_rejects_instruction_file_mode(self): + """put_object MUST raise when instruction-file mode is active.""" + mock_keyring = Mock(spec=S3Keyring) + config = S3EncryptionClientConfig(keyring=mock_keyring) + plugin = S3EncryptionClientPlugin(config) + + # Activate instruction file mode + plugin._context.instruction_file_mode = True + + params = {"body": b"test data", "headers": {}} + + with pytest.raises(S3EncryptionClientError, match="not supported in put_object"): + plugin.on_put_object_before_call(params) From 586ad4bd74ea13ce1ee995831676107293accecf Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Fri, 17 Apr 2026 17:08:38 -0700 Subject: [PATCH 66/81] chore(perf): add performance testing (#173) --- .github/workflows/all-ci.yml | 10 + .github/workflows/python-integ.yml | 2 +- .github/workflows/python-perf.yml | 66 ++++ .gitignore | 1 + Makefile | 8 +- test/performance/__init__.py | 2 + test/performance/conftest.py | 43 +++ test/performance/generate_report.py | 398 ++++++++++++++++++++ test/performance/test_perf_s3_encryption.py | 277 ++++++++++++++ 9 files changed, 804 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/python-perf.yml create mode 100644 test/performance/__init__.py create mode 100644 test/performance/conftest.py create mode 100644 test/performance/generate_report.py create mode 100644 test/performance/test_perf_s3_encryption.py diff --git a/.github/workflows/all-ci.yml b/.github/workflows/all-ci.yml index e35342fc..d8c92bb9 100644 --- a/.github/workflows/all-ci.yml +++ b/.github/workflows/all-ci.yml @@ -37,6 +37,16 @@ jobs: python-version: ${{ inputs.python-version || '3.11' }} secrets: inherit + python-perf: + permissions: + id-token: write + contents: read + name: Python Performance Tests + uses: ./.github/workflows/python-perf.yml + with: + python-version: ${{ inputs.python-version || '3.11' }} + secrets: inherit + run-duvet: permissions: id-token: write diff --git a/.github/workflows/python-integ.yml b/.github/workflows/python-integ.yml index 0f50c8e8..ec50a72c 100644 --- a/.github/workflows/python-integ.yml +++ b/.github/workflows/python-integ.yml @@ -49,7 +49,7 @@ jobs: - name: Run unit tests run: | - uv run pytest test/ --ignore=test/integration/ --verbose \ + uv run pytest test/ --ignore=test/integration/ --ignore=test/performance/ --verbose \ --cov=src/s3_encryption --cov-report=term-missing --cov-report=html:coverage-unit \ --cov-fail-under=89 diff --git a/.github/workflows/python-perf.yml b/.github/workflows/python-perf.yml new file mode 100644 index 00000000..c5a4ab41 --- /dev/null +++ b/.github/workflows/python-perf.yml @@ -0,0 +1,66 @@ +name: Python Performance Tests + +on: + workflow_call: + inputs: + python-version: + description: "Python version to use" + default: "3.11" + required: false + type: string + +jobs: + python-perf: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + submodules: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ inputs.python-version || '3.11' }} + + - name: Cache uv dependencies + uses: actions/cache@v5 + with: + path: ~/.cache/uv + key: ${{ runner.os }}-uv-${{ hashFiles('**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-uv- + + - name: Install Uv + run: pip install uv + + - name: Install dependencies + run: make install + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role + aws-region: us-west-2 + + - name: Run performance tests + run: make test-perf + env: + CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} + CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} + PERF_NUM_ROUNDS: "30" + + - name: Generate performance HTML report + if: always() + run: uv run python test/performance/generate_report.py + + - name: Upload performance report + if: always() + uses: actions/upload-artifact@v7 + with: + name: performance-report + path: perf-results/ diff --git a/.gitignore b/.gitignore index 3691eef4..b0b67407 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,4 @@ smithy-java-core/out *.pid .coverage coverage-report/ +perf-results/ diff --git a/Makefile b/Makefile index caf05737..e764a496 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: lint format test test-unit test-integration install +.PHONY: lint format test test-unit test-integration test-perf install # Default target all: lint test duvet @@ -25,12 +25,16 @@ test: test-unit test-integration test-examples # Run unit tests with coverage test-unit: - uv run pytest test/ --ignore=test/integration/ --verbose --cov=src/s3_encryption --cov-report=term-missing --cov-fail-under=89 + uv run pytest test/ --ignore=test/integration/ --ignore=test/performance/ --verbose --cov=src/s3_encryption --cov-report=term-missing --cov-fail-under=89 # Run integration tests with separate coverage test-integration: uv run pytest test/integration/ --verbose --cov=src/s3_encryption --cov-report=term-missing --cov-fail-under=83 +# Run performance tests +test-perf: + uv run pytest test/performance/ --verbose -x + test-examples: uv run pytest examples/test/ -v diff --git a/test/performance/__init__.py b/test/performance/__init__.py new file mode 100644 index 00000000..f94fd12a --- /dev/null +++ b/test/performance/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/test/performance/conftest.py b/test/performance/conftest.py new file mode 100644 index 00000000..2e686a30 --- /dev/null +++ b/test/performance/conftest.py @@ -0,0 +1,43 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Shared fixtures for performance tests.""" + +import os + +import boto3 +import pytest + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.materials.kms_keyring import KmsKeyring + +BUCKET = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") +REGION = os.environ.get("CI_AWS_REGION", "us-west-2") +KMS_KEY_ID = os.environ.get( + "CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" +) + +# Performance test configuration +NUM_ROUNDS = int(os.environ.get("PERF_NUM_ROUNDS", "30")) +OBJECT_SIZES_MB = [10, 25, 50] + + +def _make_s3ec(algorithm_suite, commitment_policy): + kms_client = boto3.client("kms", region_name=REGION) + keyring = KmsKeyring(kms_client, KMS_KEY_ID) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig( + keyring, + encryption_algorithm=algorithm_suite, + commitment_policy=commitment_policy, + ) + return S3EncryptionClient(wrapped_client, config) + + +@pytest.fixture(scope="module") +def plain_s3(): + return boto3.client("s3", region_name=REGION) + + +@pytest.fixture(scope="module") +def kms_client(): + return boto3.client("kms", region_name=REGION) diff --git a/test/performance/generate_report.py b/test/performance/generate_report.py new file mode 100644 index 00000000..14030657 --- /dev/null +++ b/test/performance/generate_report.py @@ -0,0 +1,398 @@ +#!/usr/bin/env python3 +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Generate an HTML performance report with tables, bar charts, and histograms.""" + +import json +import math +import sys +from pathlib import Path + +RESULTS_FILE = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("perf-results/results.json") +OUTPUT_FILE = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("perf-results/report.html") + +COLORS = { + "plain": "#36a2eb", + "aes_gcm": "#ff6384", + "kc_gcm": "#ff9f40", + "local": "#4bc0c0", +} + + +def _fmt(seconds: float) -> str: + if seconds < 1: + return f"{seconds * 1000:.1f} ms" + return f"{seconds:.2f} s" + + +def _percentile(sorted_vals, p): + """Compute the p-th percentile from a sorted list.""" + k = (len(sorted_vals) - 1) * (p / 100) + f = math.floor(k) + c = math.ceil(k) + if f == c: + return sorted_vals[int(k)] + return sorted_vals[f] * (c - k) + sorted_vals[c] * (k - f) + + +def _median(vals): + s = sorted(vals) + return _percentile(s, 50) + + +def _p95(vals): + s = sorted(vals) + return _percentile(s, 95) + + +def _lookup(results, prefix, size_mb): + for r in results: + if r["test"].startswith(prefix) and r["size_mb"] == size_mb: + return r + return None + + +def _bar_chart_svg(chart_id, title, groups, sizes, width=700, bar_h=28, gap=6): + """Render a grouped horizontal bar chart (median values) as SVG.""" + label_col_w = 120 + chart_w = width - label_col_w - 80 + n_groups = len(groups) + block_h = n_groups * (bar_h + gap) + 20 + total_h = len(sizes) * block_h + 60 + + max_val = max( + (v for g in groups for v in g["values"].values()), + default=1, + ) + if max_val == 0: + max_val = 1 + + svg = [ + f'', + f'{title}', + ] + lx = label_col_w + for g in groups: + svg.append(f'') + svg.append(f'{g["label"]}') + lx += len(g["label"]) * 7 + 30 + + y = 58 + for size in sizes: + svg.append( + f'{size} MB' + ) + for i, g in enumerate(groups): + val = g["values"].get(size, 0) + bw = max(2, (val / max_val) * chart_w) + by = y + i * (bar_h + gap) + svg.append( + f'' + ) + svg.append( + f'{_fmt(val)}' + ) + y += block_h + svg.append("") + return "\n".join(svg) + + +def _histogram_svg(chart_id, title, series_list, width=700, height=220, n_bins=15): + """Render overlaid histograms for multiple series as SVG. + + Args: + chart_id: unique SVG id + title: chart title + series_list: list of {label, color, durations: [float]} + width, height: SVG dimensions + n_bins: number of histogram bins + """ + # Compute global range across all series + all_vals = [d for s in series_list for d in s["durations"]] + if not all_vals: + return "" + lo = min(all_vals) + hi = max(all_vals) + if lo == hi: + hi = lo + 0.001 # avoid zero-width range + + margin_l, margin_r, margin_t, margin_b = 60, 20, 50, 40 + plot_w = width - margin_l - margin_r + plot_h = height - margin_t - margin_b + bin_width = (hi - lo) / n_bins + + # Build histogram counts for each series + histograms = [] + global_max_count = 0 + for s in series_list: + counts = [0] * n_bins + for d in s["durations"]: + idx = min(int((d - lo) / bin_width), n_bins - 1) + counts[idx] += 1 + global_max_count = max(global_max_count, max(counts)) + histograms.append(counts) + if global_max_count == 0: + global_max_count = 1 + + svg = [ + f'', + f'{title}', + ] + + # Legend + lx = margin_l + for s in series_list: + svg.append(f'') + svg.append(f'{s["label"]}') + lx += len(s["label"]) * 6 + 24 + + # Axes + ax_y = margin_t + plot_h + svg.append( + f'' + ) + svg.append( + f'' + ) + + # X-axis labels (5 ticks) + for i in range(6): + val = lo + (hi - lo) * i / 5 + x = margin_l + plot_w * i / 5 + svg.append( + f'{_fmt(val)}' + ) + + # Y-axis labels + for i in range(4): + cnt = int(global_max_count * i / 3) + y_pos = ax_y - plot_h * i / 3 + svg.append( + f'{cnt}' + ) + + # Draw bars for each series (slightly offset for overlap visibility) + bar_px = plot_w / n_bins + n_series = len(series_list) + sub_w = bar_px / n_series if n_series > 1 else bar_px * 0.8 + + for si, (s, counts) in enumerate(zip(series_list, histograms)): + for bi, cnt in enumerate(counts): + if cnt == 0: + continue + bh = (cnt / global_max_count) * plot_h + bx = margin_l + bi * bar_px + si * sub_w + by = ax_y - bh + svg.append( + f'' + ) + + svg.append("") + return "\n".join(svg) + + +def _build_charts_and_histograms(results, sizes): + """Build bar charts (median) and histograms for each category.""" + html_parts = [] + + # --- Define chart groups --- + chart_defs = [ + { + "id": "put", + "title": "PutObject: Plain S3 vs S3EC", + "series": [ + ("Plain S3", "plain", "plain_s3_put"), + ("S3EC AES_GCM", "aes_gcm", "s3ec_put_aes_gcm"), + ("S3EC KC_GCM", "kc_gcm", "s3ec_put_kc_gcm"), + ], + }, + { + "id": "get", + "title": "GetObject: Plain S3 vs S3EC", + "series": [ + ("Plain S3", "plain", "plain_s3_get"), + ("S3EC AES_GCM", "aes_gcm", "s3ec_get_aes_gcm"), + ("S3EC KC_GCM", "kc_gcm", "s3ec_get_kc_gcm"), + ], + }, + { + "id": "rt", + "title": "Roundtrip: S3EC vs Local Crypto + Plain S3", + "series": [ + ("Local Crypto + Plain S3", "local", "local_crypto_roundtrip"), + ("S3EC AES_GCM", "aes_gcm", "s3ec_roundtrip_aes_gcm"), + ("S3EC KC_GCM", "kc_gcm", "s3ec_roundtrip_kc_gcm"), + ], + }, + ] + + for cdef in chart_defs: + # Bar chart using median + groups = [] + for label, color_key, prefix in cdef["series"]: + vals = {} + for s in sizes: + r = _lookup(results, f"{prefix}_{s}mb", s) + if r: + vals[s] = _median(r["durations_s"]) + groups.append({"label": label, "color": COLORS[color_key], "values": vals}) + html_parts.append( + _bar_chart_svg(f"chart-{cdef['id']}", f"{cdef['title']} (Median)", groups, sizes) + ) + + # Histograms — one per payload size, stacked vertically + for s in sizes: + series_list = [] + for label, color_key, prefix in cdef["series"]: + r = _lookup(results, f"{prefix}_{s}mb", s) + if r: + series_list.append( + { + "label": label, + "color": COLORS[color_key], + "durations": r["durations_s"], + } + ) + if series_list: + html_parts.append( + _histogram_svg( + f"hist-{cdef['id']}-{s}mb", + f"{cdef['title']} — {s} MB Distribution", + series_list, + ) + ) + + return "\n".join(html_parts) + + +def _build_table(results): + """Build the full results table with median and p95.""" + groups: dict[str, list[dict]] = {} + for r in results: + name = r["test"] + if "roundtrip" in name or "local_crypto" in name: + cat = "Roundtrip: S3EC vs Local Crypto + Plain S3" + elif "put" in name: + cat = "PutObject: Plain S3 vs S3EC" + elif "get" in name: + cat = "GetObject: Plain S3 vs S3EC" + else: + cat = "Other" + groups.setdefault(cat, []).append(r) + + sections_html = "" + for cat, items in groups.items(): + rows = "" + for r in sorted(items, key=lambda x: (x["size_mb"], x["test"])): + d = r["durations_s"] + med = _median(d) + p95 = _p95(d) + durations_str = ", ".join(_fmt(v) for v in d) + rows += f""" + + {r['test']} + {r['size_mb']} MB + {r['rounds']} + {_fmt(med)} + {_fmt(r['mean_s'])} + {_fmt(p95)} + {_fmt(r['min_s'])} + {_fmt(r['max_s'])} + {durations_str} + """ + + sections_html += f""" +

{cat}

+ + + + + + + + + {rows} + +
TestSizeRoundsMedianMeanp95MinMaxAll Durations
""" + return sections_html + + +def generate_html(data: dict) -> str: + config = data["config"] + results = data["results"] + timestamp = data["timestamp"] + sizes = config["object_sizes_mb"] + + visuals_html = _build_charts_and_histograms(results, sizes) + tables_html = _build_table(results) + + return f""" + + + +S3EC Performance Report + + + +

S3 Encryption Client — Performance Report

+
+ Generated: {timestamp}
+ Rounds per test: {config['num_rounds']} · + Object sizes: {', '.join(str(s) + ' MB' for s in sizes)} · + Bucket: {config['bucket']} · Region: {config['region']} +
+ +
+{visuals_html} +
+ +{tables_html} + +""" + + +def main(): + if not RESULTS_FILE.exists(): + print(f"Results file not found: {RESULTS_FILE}", file=sys.stderr) + sys.exit(1) + + with open(RESULTS_FILE) as f: + data = json.load(f) + + html = generate_html(data) + OUTPUT_FILE.parent.mkdir(parents=True, exist_ok=True) + OUTPUT_FILE.write_text(html) + print(f"Report written to {OUTPUT_FILE}") + + +if __name__ == "__main__": + main() diff --git a/test/performance/test_perf_s3_encryption.py b/test/performance/test_perf_s3_encryption.py new file mode 100644 index 00000000..590a0eb2 --- /dev/null +++ b/test/performance/test_perf_s3_encryption.py @@ -0,0 +1,277 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Performance tests comparing S3EC against plaintext S3 and local encryption + S3 upload. + +Each test runs multiple rounds with large objects to get a meaningful signal. +To control for temporal network variation, all variants within a test are +interleaved: round N of every variant runs back-to-back before moving to +round N+1. This ensures each variant experiences the same network conditions. + +Results are collected via a module-scoped list and written to a JSON file +that the HTML report generator consumes. +""" + +import json +import os +import random +import time +from datetime import datetime + +import pytest +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy + +from .conftest import BUCKET, KMS_KEY_ID, NUM_ROUNDS, OBJECT_SIZES_MB, REGION, _make_s3ec + +PERF_KEY_PREFIX = "perf-test/" +RESULTS_FILE = os.environ.get("PERF_RESULTS_FILE", "perf-results/results.json") + +_results: list[dict] = [] + +# Pre-generate payloads once at module level +_PAYLOADS: dict[int, bytes] = {} +_WARMUP_PAYLOAD = b"x" * 1024 + +# Algorithm suite configs +_AES_GCM = ( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, +) +_KC_GCM = ( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, +) + + +def _get_payload(size_mb: int) -> bytes: + if size_mb not in _PAYLOADS: + chunk = os.urandom(1024) + _PAYLOADS[size_mb] = (chunk * 1024 * size_mb)[: size_mb * 1024 * 1024] + return _PAYLOADS[size_mb] + + +def _unique_key(prefix: str) -> str: + return PERF_KEY_PREFIX + prefix + datetime.now().strftime("%Y%m%d-%H%M%S-%f") + + +def _record(test_name, size_mb, durations): + _results.append( + { + "test": test_name, + "size_mb": size_mb, + "rounds": len(durations), + "durations_s": durations, + "mean_s": sum(durations) / len(durations), + "min_s": min(durations), + "max_s": max(durations), + } + ) + + +def _warmup_connection(client): + """Warm up TCP/TLS connections with a tiny payload.""" + key = _unique_key("warmup-conn-") + client.put_object(Bucket=BUCKET, Key=key, Body=_WARMUP_PAYLOAD) + resp = client.get_object(Bucket=BUCKET, Key=key) + resp["Body"].read() + + +# --------------------------------------------------------------------------- +# Interleaved put_object benchmark +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("size_mb", OBJECT_SIZES_MB) +def test_put_interleaved(plain_s3, size_mb): + """Interleaved put_object: plain S3, S3EC AES_GCM, S3EC KC_GCM.""" + payload = _get_payload(size_mb) + + s3ec_aes = _make_s3ec(*_AES_GCM) + s3ec_kc = _make_s3ec(*_KC_GCM) + + # Warm up all connections + _warmup_connection(plain_s3) + _warmup_connection(s3ec_aes) + _warmup_connection(s3ec_kc) + + plain_d, aes_d, kc_d = [], [], [] + + # Define the three variants as callables + def run_plain(): + key = _unique_key(f"plain-put-{size_mb}mb-") + t0 = time.perf_counter() + plain_s3.put_object(Bucket=BUCKET, Key=key, Body=payload) + return time.perf_counter() - t0 + + def run_aes(): + key = _unique_key(f"s3ec-put-aes-{size_mb}mb-") + t0 = time.perf_counter() + s3ec_aes.put_object(Bucket=BUCKET, Key=key, Body=payload) + return time.perf_counter() - t0 + + def run_kc(): + key = _unique_key(f"s3ec-put-kc-{size_mb}mb-") + t0 = time.perf_counter() + s3ec_kc.put_object(Bucket=BUCKET, Key=key, Body=payload) + return time.perf_counter() - t0 + + variants = [(run_plain, plain_d), (run_aes, aes_d), (run_kc, kc_d)] + + for _ in range(NUM_ROUNDS): + # Shuffle order each round to eliminate positional bias + random.shuffle(variants) + for fn, collector in variants: + collector.append(fn()) + + _record(f"plain_s3_put_{size_mb}mb", size_mb, plain_d) + _record(f"s3ec_put_aes_gcm_{size_mb}mb", size_mb, aes_d) + _record(f"s3ec_put_kc_gcm_{size_mb}mb", size_mb, kc_d) + + +# --------------------------------------------------------------------------- +# Interleaved get_object benchmark +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("size_mb", OBJECT_SIZES_MB) +def test_get_interleaved(plain_s3, size_mb): + """Interleaved get_object: plain S3, S3EC AES_GCM, S3EC KC_GCM.""" + payload = _get_payload(size_mb) + + s3ec_aes = _make_s3ec(*_AES_GCM) + s3ec_kc = _make_s3ec(*_KC_GCM) + + # Upload source objects + plain_key = _unique_key(f"plain-get-src-{size_mb}mb-") + plain_s3.put_object(Bucket=BUCKET, Key=plain_key, Body=payload) + + aes_key = _unique_key(f"s3ec-get-src-aes-{size_mb}mb-") + s3ec_aes.put_object(Bucket=BUCKET, Key=aes_key, Body=payload) + + kc_key = _unique_key(f"s3ec-get-src-kc-{size_mb}mb-") + s3ec_kc.put_object(Bucket=BUCKET, Key=kc_key, Body=payload) + + # Warm up all connections + _warmup_connection(plain_s3) + _warmup_connection(s3ec_aes) + _warmup_connection(s3ec_kc) + + plain_d, aes_d, kc_d = [], [], [] + + def run_plain(): + t0 = time.perf_counter() + resp = plain_s3.get_object(Bucket=BUCKET, Key=plain_key) + resp["Body"].read() + return time.perf_counter() - t0 + + def run_aes(): + t0 = time.perf_counter() + resp = s3ec_aes.get_object(Bucket=BUCKET, Key=aes_key) + resp["Body"].read() + return time.perf_counter() - t0 + + def run_kc(): + t0 = time.perf_counter() + resp = s3ec_kc.get_object(Bucket=BUCKET, Key=kc_key) + resp["Body"].read() + return time.perf_counter() - t0 + + variants = [(run_plain, plain_d), (run_aes, aes_d), (run_kc, kc_d)] + + for _ in range(NUM_ROUNDS): + random.shuffle(variants) + for fn, collector in variants: + collector.append(fn()) + + _record(f"plain_s3_get_{size_mb}mb", size_mb, plain_d) + _record(f"s3ec_get_aes_gcm_{size_mb}mb", size_mb, aes_d) + _record(f"s3ec_get_kc_gcm_{size_mb}mb", size_mb, kc_d) + + +# --------------------------------------------------------------------------- +# Interleaved roundtrip benchmark +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("size_mb", OBJECT_SIZES_MB) +def test_roundtrip_interleaved(plain_s3, kms_client, size_mb): + """Interleaved roundtrip: S3EC AES_GCM, S3EC KC_GCM, local crypto + plain S3.""" + payload = _get_payload(size_mb) + + s3ec_aes = _make_s3ec(*_AES_GCM) + s3ec_kc = _make_s3ec(*_KC_GCM) + + # Warm up all connections + _warmup_connection(plain_s3) + _warmup_connection(s3ec_aes) + _warmup_connection(s3ec_kc) + kms_client.generate_data_key(KeyId=KMS_KEY_ID, KeySpec="AES_256") + + aes_d, kc_d, local_d = [], [], [] + + def run_aes(): + key = _unique_key(f"s3ec-rt-aes-{size_mb}mb-") + t0 = time.perf_counter() + s3ec_aes.put_object(Bucket=BUCKET, Key=key, Body=payload) + resp = s3ec_aes.get_object(Bucket=BUCKET, Key=key) + resp["Body"].read() + return time.perf_counter() - t0 + + def run_kc(): + key = _unique_key(f"s3ec-rt-kc-{size_mb}mb-") + t0 = time.perf_counter() + s3ec_kc.put_object(Bucket=BUCKET, Key=key, Body=payload) + resp = s3ec_kc.get_object(Bucket=BUCKET, Key=key) + resp["Body"].read() + return time.perf_counter() - t0 + + def run_local(): + key = _unique_key(f"local-rt-{size_mb}mb-") + t0 = time.perf_counter() + dk_resp = kms_client.generate_data_key(KeyId=KMS_KEY_ID, KeySpec="AES_256") + aesgcm = AESGCM(dk_resp["Plaintext"]) + nonce = os.urandom(12) + ciphertext = aesgcm.encrypt(nonce, payload, None) + plain_s3.put_object(Bucket=BUCKET, Key=key, Body=nonce + ciphertext) + resp = plain_s3.get_object(Bucket=BUCKET, Key=key) + blob = resp["Body"].read() + aesgcm.decrypt(blob[:12], blob[12:], None) + return time.perf_counter() - t0 + + variants = [(run_aes, aes_d), (run_kc, kc_d), (run_local, local_d)] + + for _ in range(NUM_ROUNDS): + random.shuffle(variants) + for fn, collector in variants: + collector.append(fn()) + + _record(f"s3ec_roundtrip_aes_gcm_{size_mb}mb", size_mb, aes_d) + _record(f"s3ec_roundtrip_kc_gcm_{size_mb}mb", size_mb, kc_d) + _record(f"local_crypto_roundtrip_{size_mb}mb", size_mb, local_d) + + +# --------------------------------------------------------------------------- +# Write results to JSON at end of module +# --------------------------------------------------------------------------- + + +def test_zz_write_results(): + """Final test that writes collected results to a JSON file for the HTML report.""" + os.makedirs(os.path.dirname(RESULTS_FILE) or ".", exist_ok=True) + with open(RESULTS_FILE, "w") as f: + json.dump( + { + "timestamp": datetime.now().isoformat(), + "config": { + "num_rounds": NUM_ROUNDS, + "object_sizes_mb": OBJECT_SIZES_MB, + "bucket": BUCKET, + "region": REGION, + }, + "results": _results, + }, + f, + indent=2, + ) + print(f"\nPerformance results written to {RESULTS_FILE}") From 515aa4b70d8a3fc2e560a18da2d3dbb899ff489e Mon Sep 17 00:00:00 2001 From: Tony Knapp <5892063+texastony@users.noreply.github.com> Date: Mon, 20 Apr 2026 17:13:14 -0700 Subject: [PATCH 67/81] feat: delete_object and delete_objects (#175) * feat: implement delete_object on S3EncryptionClient Implement delete_object per the spec requirement that DeleteObject MUST delete both the given object key and its associated instruction file. Accepts an optional InstructionFileSuffix kwarg (default ".instruction") mirroring get_object's per-request suffix pattern. * feat: implement delete_objects API Implement DeleteObjects on S3EncryptionClient per spec requirements: - DeleteObjects MUST delete each of the given objects - DeleteObjects MUST delete each of the corresponding instruction files using the default instruction file suffix Uses two separate delete_objects calls (objects, then instruction files) to preserve the S3 1,000-key limit for callers and keep the response clean. Follows the same pattern as the existing delete_object method. * Add integration tests for delete_objects API --- src/s3_encryption/__init__.py | 87 ++++++++ .../test_i_s3_encryption_delete_objects.py | 126 ++++++++++++ test/test_s3_encryption_client_delete.py | 108 ++++++++++ ...est_s3_encryption_client_delete_objects.py | 188 ++++++++++++++++++ 4 files changed, 509 insertions(+) create mode 100644 test/integration/test_i_s3_encryption_delete_objects.py create mode 100644 test/test_s3_encryption_client_delete.py create mode 100644 test/test_s3_encryption_client_delete_objects.py diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index a9d5a3a1..f684c6a9 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -395,6 +395,93 @@ def put_object(self, **kwargs): if hasattr(self._plugin._context, _CTX_ENCRYPTION_CONTEXT): delattr(self._plugin._context, _CTX_ENCRYPTION_CONTEXT) + ##= specification/s3-encryption/client.md#required-api-operations + ##% - DeleteObject MUST be implemented by the S3EC. + def delete_object(self, **kwargs): + """Delete an object and its associated instruction file from S3. + + Args: + **kwargs: Arguments to pass to the S3 client's delete_object method. + Must include Bucket and Key parameters. + May include InstructionFileSuffix to override the default + ".instruction" suffix for instruction file deletion. + + Returns: + The response from the S3 client's delete_object call for the object. + + Raises: + S3EncryptionClientError: If the delete operation fails. + """ + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=implementation + ##% The default Instruction File behavior uses the same S3 object key + ##% as its associated object suffixed with ".instruction". + instruction_file_suffix = kwargs.pop("InstructionFileSuffix", ".instruction") + + try: + ##= specification/s3-encryption/client.md#required-api-operations + ##= type=implementation + ##% - DeleteObject MUST delete the given object key. + response = self.wrapped_s3_client.delete_object(**kwargs) + + ##= specification/s3-encryption/client.md#required-api-operations + ##= type=implementation + ##% - DeleteObject MUST delete the associated instruction file + ##% using the default instruction file suffix. + instruction_key = kwargs["Key"] + instruction_file_suffix + self.wrapped_s3_client.delete_object(Bucket=kwargs["Bucket"], Key=instruction_key) + + return response + except S3EncryptionClientError: + raise + except Exception as e: + raise S3EncryptionClientError(f"Failed to delete object: {str(e)}") from e + + ##= specification/s3-encryption/client.md#required-api-operations + ##% - DeleteObjects MUST be implemented by the S3EC. + def delete_objects(self, **kwargs): + """Delete multiple objects and their associated instruction files from S3. + + 2 requests are issued, one for the objects, and one for the instruction files. + If either requests fail, the operation fails, and maybe tried again to clean up any missed files. + + Args: + **kwargs: Arguments to pass to the S3 client's delete_objects method. + Must include Bucket and Delete (with Objects list) parameters. + May include InstructionFileSuffix to override the default + ".instruction" suffix for instruction file deletion. + + Returns: + The response from the S3 client's delete_objects call for the objects. + + Raises: + S3EncryptionClientError: If either delete operations fails. + """ + instruction_file_suffix = kwargs.pop("InstructionFileSuffix", ".instruction") + + try: + ##= specification/s3-encryption/client.md#required-api-operations + ##= type=implementation + ##% - DeleteObjects MUST delete each of the given objects. + response = self.wrapped_s3_client.delete_objects(**kwargs) + + ##= specification/s3-encryption/client.md#required-api-operations + ##= type=implementation + ##% - DeleteObjects MUST delete each of the corresponding instruction files + ##% using the default instruction file suffix. + instruction_objects = [ + {"Key": obj["Key"] + instruction_file_suffix} for obj in kwargs["Delete"]["Objects"] + ] + self.wrapped_s3_client.delete_objects( + Bucket=kwargs["Bucket"], Delete={"Objects": instruction_objects} + ) + + return response + except S3EncryptionClientError: + raise + except Exception as e: + raise S3EncryptionClientError(f"Failed to delete objects: {str(e)}") from e + def get_object(self, **kwargs): """Download and decrypt an object from S3. diff --git a/test/integration/test_i_s3_encryption_delete_objects.py b/test/integration/test_i_s3_encryption_delete_objects.py new file mode 100644 index 00000000..2d6c7876 --- /dev/null +++ b/test/integration/test_i_s3_encryption_delete_objects.py @@ -0,0 +1,126 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Integration tests for S3EncryptionClient.delete_objects.""" + +import os +from datetime import datetime + +import boto3 +import pytest +from botocore.exceptions import ClientError + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy + +bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") +region = os.environ.get("CI_AWS_REGION", "us-west-2") +kms_key_id = os.environ.get( + "CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" +) + +ALGORITHM_CONFIGS = [ + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + id="AES_GCM", + ), + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + id="KC_GCM", + ), +] + + +def _make_client(algorithm_suite, commitment_policy): + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig( + keyring, + encryption_algorithm=algorithm_suite, + commitment_policy=commitment_policy, + ) + return S3EncryptionClient(wrapped_client, config) + + +def _unique_key(prefix): + return prefix + datetime.now().strftime("%Y-%m-%d-%H:%M:%S-%f") + + +def _object_exists(key): + """Return True if the object exists in the test bucket.""" + s3 = boto3.client("s3") + try: + s3.head_object(Bucket=bucket, Key=key) + return True + except ClientError as e: + if e.response["Error"]["Code"] == "404": + return False + raise + + +##= specification/s3-encryption/client.md#required-api-operations +##= type=test +##% - DeleteObjects MUST delete each of the given objects. +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_delete_objects_deletes_objects(algorithm_suite, commitment_policy): + """delete_objects removes the encrypted objects from S3.""" + s3ec = _make_client(algorithm_suite, commitment_policy) + keys = [_unique_key("del-objs-"), _unique_key("del-objs-")] + + for key in keys: + s3ec.put_object(Bucket=bucket, Key=key, Body=b"data") + + s3ec.delete_objects( + Bucket=bucket, + Delete={"Objects": [{"Key": k} for k in keys]}, + ) + + for key in keys: + assert not _object_exists(key) + + +##= specification/s3-encryption/client.md#required-api-operations +##= type=test +##% - DeleteObjects MUST delete each of the corresponding instruction files +##% using the default instruction file suffix. +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_delete_objects_deletes_instruction_files(algorithm_suite, commitment_policy): + """delete_objects also removes the .instruction files from S3.""" + s3ec = _make_client(algorithm_suite, commitment_policy) + keys = [_unique_key("del-objs-instr-"), _unique_key("del-objs-instr-")] + + # Put instruction-file-based objects by uploading instruction files manually + plain_s3 = boto3.client("s3") + for key in keys: + s3ec.put_object(Bucket=bucket, Key=key, Body=b"data") + # Also create a fake instruction file to verify it gets deleted + plain_s3.put_object(Bucket=bucket, Key=key + ".instruction", Body=b"{}") + + s3ec.delete_objects( + Bucket=bucket, + Delete={"Objects": [{"Key": k} for k in keys]}, + ) + + for key in keys: + assert not _object_exists(key) + assert not _object_exists(key + ".instruction") + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_delete_objects_returns_response(algorithm_suite, commitment_policy): + """delete_objects returns the S3 response from the object deletion.""" + s3ec = _make_client(algorithm_suite, commitment_policy) + key = _unique_key("del-objs-resp-") + s3ec.put_object(Bucket=bucket, Key=key, Body=b"data") + + response = s3ec.delete_objects( + Bucket=bucket, + Delete={"Objects": [{"Key": key}]}, + ) + + assert "Deleted" in response + deleted_keys = [d["Key"] for d in response["Deleted"]] + assert key in deleted_keys diff --git a/test/test_s3_encryption_client_delete.py b/test/test_s3_encryption_client_delete.py new file mode 100644 index 00000000..1279abab --- /dev/null +++ b/test/test_s3_encryption_client_delete.py @@ -0,0 +1,108 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for S3EncryptionClient.delete_object.""" + +from unittest.mock import Mock, call + +import pytest + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.materials.keyring import S3Keyring + + +def _make_client(): + """Create an S3EncryptionClient with a mocked wrapped S3 client.""" + mock_keyring = Mock(spec=S3Keyring) + mock_s3 = Mock() + mock_s3.meta.events = Mock() + config = S3EncryptionClientConfig(keyring=mock_keyring) + s3ec = S3EncryptionClient(wrapped_s3_client=mock_s3, config=config) + return s3ec, mock_s3 + + +class TestDeleteObject: + ##= specification/s3-encryption/client.md#required-api-operations + ##= type=test + ##% - DeleteObject MUST delete the given object key. + def test_deletes_object(self): + """delete_object forwards the call to the wrapped client.""" + s3ec, mock_s3 = _make_client() + mock_s3.delete_object.return_value = {"DeleteMarker": True} + + response = s3ec.delete_object(Bucket="bucket", Key="key") + + assert response == {"DeleteMarker": True} + assert mock_s3.delete_object.call_args_list[0] == call(Bucket="bucket", Key="key") + + ##= specification/s3-encryption/client.md#required-api-operations + ##= type=test + ##% - DeleteObject MUST delete the associated instruction file + ##% using the default instruction file suffix. + def test_deletes_instruction_file(self): + """delete_object also deletes the instruction file with default suffix.""" + s3ec, mock_s3 = _make_client() + + s3ec.delete_object(Bucket="bucket", Key="key") + + assert mock_s3.delete_object.call_count == 2 + assert mock_s3.delete_object.call_args_list[1] == call( + Bucket="bucket", Key="key.instruction" + ) + + def test_returns_object_delete_response(self): + """delete_object returns the response from the object deletion, not the instruction file.""" + s3ec, mock_s3 = _make_client() + object_response = {"DeleteMarker": True, "VersionId": "v1"} + instruction_response = {"DeleteMarker": False, "VersionId": "v2"} + mock_s3.delete_object.side_effect = [object_response, instruction_response] + + response = s3ec.delete_object(Bucket="bucket", Key="key") + + assert response == object_response + + def test_wraps_unexpected_errors(self): + """delete_object wraps unexpected errors in S3EncryptionClientError.""" + s3ec, mock_s3 = _make_client() + mock_s3.delete_object.side_effect = RuntimeError("network error") + + with pytest.raises(S3EncryptionClientError, match="Failed to delete object"): + s3ec.delete_object(Bucket="bucket", Key="key") + + def test_reraises_s3_encryption_client_error(self): + """delete_object re-raises S3EncryptionClientError without wrapping.""" + s3ec, mock_s3 = _make_client() + mock_s3.delete_object.side_effect = S3EncryptionClientError("original error") + + with pytest.raises(S3EncryptionClientError, match="original error"): + s3ec.delete_object(Bucket="bucket", Key="key") + + def test_passes_extra_kwargs(self): + """delete_object forwards extra kwargs like VersionId to the wrapped client.""" + s3ec, mock_s3 = _make_client() + + s3ec.delete_object(Bucket="bucket", Key="key", VersionId="abc123") + + assert mock_s3.delete_object.call_args_list[0] == call( + Bucket="bucket", Key="key", VersionId="abc123" + ) + + def test_custom_instruction_file_suffix(self): + """delete_object uses a custom instruction file suffix when provided.""" + s3ec, mock_s3 = _make_client() + + s3ec.delete_object(Bucket="bucket", Key="key", InstructionFileSuffix=".custom-suffix") + + assert mock_s3.delete_object.call_count == 2 + assert mock_s3.delete_object.call_args_list[1] == call( + Bucket="bucket", Key="key.custom-suffix" + ) + + def test_instruction_file_suffix_not_forwarded_to_s3(self): + """InstructionFileSuffix is popped from kwargs and not sent to S3.""" + s3ec, mock_s3 = _make_client() + + s3ec.delete_object(Bucket="bucket", Key="key", InstructionFileSuffix=".custom") + + # First call (object delete) should not contain InstructionFileSuffix + assert mock_s3.delete_object.call_args_list[0] == call(Bucket="bucket", Key="key") diff --git a/test/test_s3_encryption_client_delete_objects.py b/test/test_s3_encryption_client_delete_objects.py new file mode 100644 index 00000000..c1045ca3 --- /dev/null +++ b/test/test_s3_encryption_client_delete_objects.py @@ -0,0 +1,188 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for S3EncryptionClient.delete_objects.""" + +from unittest.mock import Mock, call + +import pytest + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.materials.keyring import S3Keyring + + +def _make_client(): + """Create an S3EncryptionClient with a mocked wrapped S3 client.""" + mock_keyring = Mock(spec=S3Keyring) + mock_s3 = Mock() + mock_s3.meta.events = Mock() + config = S3EncryptionClientConfig(keyring=mock_keyring) + s3ec = S3EncryptionClient(wrapped_s3_client=mock_s3, config=config) + return s3ec, mock_s3 + + +class TestDeleteObjects: + ##= specification/s3-encryption/client.md#required-api-operations + ##= type=test + ##% - DeleteObjects MUST delete each of the given objects. + def test_deletes_objects(self): + """delete_objects forwards the Delete parameter to the wrapped client.""" + s3ec, mock_s3 = _make_client() + mock_s3.delete_objects.return_value = { + "Deleted": [{"Key": "key1"}, {"Key": "key2"}], + } + + delete_param = {"Objects": [{"Key": "key1"}, {"Key": "key2"}]} + response = s3ec.delete_objects(Bucket="bucket", Delete=delete_param) + + assert response == {"Deleted": [{"Key": "key1"}, {"Key": "key2"}]} + assert mock_s3.delete_objects.call_args_list[0] == call( + Bucket="bucket", Delete=delete_param + ) + + ##= specification/s3-encryption/client.md#required-api-operations + ##= type=test + ##% - DeleteObjects MUST delete each of the corresponding instruction files + ##% using the default instruction file suffix. + def test_deletes_instruction_files(self): + """delete_objects also deletes instruction files for each object.""" + s3ec, mock_s3 = _make_client() + mock_s3.delete_objects.return_value = {"Deleted": []} + + s3ec.delete_objects( + Bucket="bucket", + Delete={"Objects": [{"Key": "key1"}, {"Key": "key2"}]}, + ) + + assert mock_s3.delete_objects.call_count == 2 + assert mock_s3.delete_objects.call_args_list[1] == call( + Bucket="bucket", + Delete={ + "Objects": [ + {"Key": "key1.instruction"}, + {"Key": "key2.instruction"}, + ], + }, + ) + + def test_returns_object_delete_response(self): + """delete_objects returns the response from the object deletion, not the instruction file deletion.""" + s3ec, mock_s3 = _make_client() + object_response = {"Deleted": [{"Key": "key1"}]} + instruction_response = {"Deleted": [{"Key": "key1.instruction"}]} + mock_s3.delete_objects.side_effect = [object_response, instruction_response] + + response = s3ec.delete_objects( + Bucket="bucket", + Delete={"Objects": [{"Key": "key1"}]}, + ) + + assert response == object_response + + def test_wraps_unexpected_errors(self): + """delete_objects wraps unexpected errors in S3EncryptionClientError.""" + s3ec, mock_s3 = _make_client() + mock_s3.delete_objects.side_effect = RuntimeError("network error") + + with pytest.raises(S3EncryptionClientError, match="Failed to delete objects"): + s3ec.delete_objects( + Bucket="bucket", + Delete={"Objects": [{"Key": "key1"}]}, + ) + + def test_reraises_s3_encryption_client_error(self): + """delete_objects re-raises S3EncryptionClientError without wrapping.""" + s3ec, mock_s3 = _make_client() + mock_s3.delete_objects.side_effect = S3EncryptionClientError("original error") + + with pytest.raises(S3EncryptionClientError, match="original error"): + s3ec.delete_objects( + Bucket="bucket", + Delete={"Objects": [{"Key": "key1"}]}, + ) + + def test_passes_extra_kwargs(self): + """delete_objects forwards extra kwargs to the wrapped client.""" + s3ec, mock_s3 = _make_client() + mock_s3.delete_objects.return_value = {"Deleted": []} + + s3ec.delete_objects( + Bucket="bucket", + Delete={"Objects": [{"Key": "key1"}]}, + RequestPayer="requester", + ) + + assert mock_s3.delete_objects.call_args_list[0] == call( + Bucket="bucket", + Delete={"Objects": [{"Key": "key1"}]}, + RequestPayer="requester", + ) + + def test_custom_instruction_file_suffix(self): + """delete_objects uses a custom instruction file suffix when provided.""" + s3ec, mock_s3 = _make_client() + mock_s3.delete_objects.return_value = {"Deleted": []} + + s3ec.delete_objects( + Bucket="bucket", + Delete={"Objects": [{"Key": "key1"}]}, + InstructionFileSuffix=".custom-suffix", + ) + + assert mock_s3.delete_objects.call_count == 2 + assert mock_s3.delete_objects.call_args_list[1] == call( + Bucket="bucket", + Delete={"Objects": [{"Key": "key1.custom-suffix"}]}, + ) + + def test_instruction_file_suffix_not_forwarded_to_s3(self): + """InstructionFileSuffix is popped from kwargs and not sent to S3.""" + s3ec, mock_s3 = _make_client() + mock_s3.delete_objects.return_value = {"Deleted": []} + + s3ec.delete_objects( + Bucket="bucket", + Delete={"Objects": [{"Key": "key1"}]}, + InstructionFileSuffix=".custom", + ) + + assert mock_s3.delete_objects.call_args_list[0] == call( + Bucket="bucket", + Delete={"Objects": [{"Key": "key1"}]}, + ) + + def test_preserves_version_ids_in_objects(self): + """delete_objects preserves VersionId in the Objects list.""" + s3ec, mock_s3 = _make_client() + mock_s3.delete_objects.return_value = {"Deleted": []} + + s3ec.delete_objects( + Bucket="bucket", + Delete={ + "Objects": [ + {"Key": "key1", "VersionId": "v1"}, + {"Key": "key2", "VersionId": "v2"}, + ] + }, + ) + + # First call preserves VersionId + assert mock_s3.delete_objects.call_args_list[0] == call( + Bucket="bucket", + Delete={ + "Objects": [ + {"Key": "key1", "VersionId": "v1"}, + {"Key": "key2", "VersionId": "v2"}, + ] + }, + ) + # Instruction file call does NOT include VersionId + assert mock_s3.delete_objects.call_args_list[1] == call( + Bucket="bucket", + Delete={ + "Objects": [ + {"Key": "key1.instruction"}, + {"Key": "key2.instruction"}, + ], + }, + ) From 7ca15e833dabd2e29477201764be2c280391c378 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Mon, 27 Apr 2026 13:25:34 -0700 Subject: [PATCH 68/81] chore(tests): add security tests (#174) * chore: add tests around downgrade and EC tampering --- src/s3_encryption/decryptor.py | 7 +- src/s3_encryption/materials/kms_keyring.py | 10 + src/s3_encryption/pipelines.py | 39 +- .../amazon/encryption/s3/RoundTripTests.java | 4 + .../amazon/encryption/s3/TestUtils.java | 5 + test/integration/test_i_security.py | 630 ++++++++++++++++++ test/test_kms_keyring.py | 55 ++ test/test_pipelines.py | 35 + 8 files changed, 781 insertions(+), 4 deletions(-) create mode 100644 test/integration/test_i_security.py diff --git a/src/s3_encryption/decryptor.py b/src/s3_encryption/decryptor.py index e3d4eece..f76c19dc 100644 --- a/src/s3_encryption/decryptor.py +++ b/src/s3_encryption/decryptor.py @@ -72,8 +72,11 @@ def finalize(self, data: bytes) -> bytes: plaintext = self._decryptor.update(data) if data else b"" plaintext += self._decryptor.finalize() return self._unpadder.update(plaintext) + self._unpadder.finalize() - except Exception as e: - raise S3EncryptionClientSecurityError(f"Failed to decrypt CBC content: {e}") from e + except Exception: + # Use a fixed message for all CBC failures to prevent padding oracle attacks. + # Different failure modes (bad padding, truncated ciphertext, wrong key) MUST + # produce identical error responses so an attacker cannot distinguish them. + raise S3EncryptionClientSecurityError("Failed to decrypt CBC content.") from None @define diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index edf1d27b..abd6fad4 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -200,6 +200,16 @@ def on_decrypt(self, dec_materials, encrypted_data_keys=None): f"Enable legacy wrapping algorithms to use legacy key wrapping " f"algorithm: {edk.key_provider_info}" ) + # The KmsV1 wrapping algorithm does not support caller-provided + # encryption context. If the caller provided encryption context, + # the client MUST reject the request. This prevents a downgrade + # from kms+context to kms from bypassing context validation. + if dec_materials.encryption_context_from_request: + raise S3EncryptionClientError( + "Encryption context is not supported with the KmsV1 (kms) " + "wrapping algorithm. Use kms+context wrapping algorithm to " + "use encryption context." + ) else: raise S3EncryptionClientError( f"{edk.key_provider_info} is not a valid key wrapping algorithm!" diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 0a107a5f..c4fe4867 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -316,6 +316,30 @@ def decrypt( # Determine the algorithm suite from the metadata algorithm_suite = self._determine_algorithm_suite(metadata) + # Reject metadata that contains keys from multiple format versions. + # This prevents format confusion attacks where an attacker injects + # V2 keys via an instruction file to bypass V3 key-commitment verification. + if metadata.has_exclusive_key_collision(): + raise S3EncryptionClientError( + "Object metadata contains keys from multiple format versions. " + "The object or its instruction file may have been tampered with." + ) + + # Also reject V2 format metadata that contains V3 content keys. + # In the instruction file injection scenario, the attacker replaces + # V3 EDK keys with V2 keys, but V3 content keys (x-amz-c, x-amz-d, + # x-amz-i) remain from the object metadata. This combination is + # never produced by legitimate encryption. + if metadata.is_v2_format() and ( + metadata.content_cipher_v3 is not None + or metadata.key_commitment_v3 is not None + or metadata.message_id_v3 is not None + ): + raise S3EncryptionClientError( # pragma: no cover — only reachable via instruction file merge; covered by TestInstructionFileFormatConfusion + "Object metadata contains V2 format keys alongside V3 content keys. " + "The object or its instruction file may have been tampered with." + ) + # Determine which format we're dealing with and get decryption materials if metadata.is_v1_format(): dec_materials = self._decrypt_v1(metadata, encryption_context) @@ -590,7 +614,13 @@ def _decrypt_v3(self, metadata, encryption_context) -> DecryptionMaterials: # Map V3 compressed wrapping algorithm to canonical key_provider_info raw_wrap_alg = metadata.encrypted_data_key_algorithm_v3 or "12" - wrap_alg = self._V3_WRAP_ALG_MAP.get(raw_wrap_alg, raw_wrap_alg) + wrap_alg = self._V3_WRAP_ALG_MAP.get(raw_wrap_alg) + if wrap_alg is None: + raise S3EncryptionClientError( + f"Unknown V3 wrapping algorithm: '{raw_wrap_alg}'. " + f"Valid values are: {list(self._V3_WRAP_ALG_MAP.keys())}. " + f"The object metadata may have been tampered with." + ) encrypted_data_key = EncryptedDataKey( key_provider_id=b"S3Keyring", @@ -607,8 +637,13 @@ def _decrypt_v3(self, metadata, encryption_context) -> DecryptionMaterials: stored_context = {} if wrap_alg == "kms+context": raw_ctx = metadata.encryption_context_v3 - else: + elif wrap_alg in ("AES/GCM", "RSA-OAEP-SHA1"): raw_ctx = metadata.mat_desc_v3 + else: + raise S3EncryptionClientError( # pragma: no cover — defense in depth, unreachable + f"Unexpected V3 wrapping algorithm for context selection: '{wrap_alg}'. " + f"The object metadata may have been tampered with." + ) if raw_ctx is not None: if isinstance(raw_ctx, dict): diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index e6cfae84..4763663d 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -332,6 +332,10 @@ public void kmsV1Legacy(TestUtils.LanguageServerTarget language) { @ParameterizedTest(name = "{displayName} for Encrypt: Java, Decrypt: {0}") @MethodSource("software.amazon.encryption.s3.TestUtils#clientsForTest") public void kmsV1LegacyWithEncCtx(TestUtils.LanguageServerTarget language) { + if (KMSV1_ENCRYPTION_CONTEXT_UNSUPPORTED.contains(language.getLanguageName())) { + throw new TestAbortedException( + "KmsV1 with encryption context not supported for: " + language.getLanguageName()); + } S3ECTestServerClient client = testServerClientFor(language); final String objectKey = appendTestSuffix("test-key-kms-v1-with-enc-ctx-" + language); final String input = "simple-test-input"; diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index d488fd2d..c1464eaf 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -98,6 +98,11 @@ public class TestUtils { public static final Set ENCRYPTION_CONTEXT_ON_ENCRYPT_UNSUPPORTED = Set.of(NET_V3_TRANSITION, NET_V4); + // Languages that reject caller-provided encryption context when the + // wrapping algorithm is KmsV1 ("kms"). + public static final Set KMSV1_ENCRYPTION_CONTEXT_UNSUPPORTED = + Set.of(PYTHON_V3); + public static final Set RE_ENCRYPT_SUPPORTED = Set.of(JAVA_V3_TRANSITION, JAVA_V4); diff --git a/test/integration/test_i_security.py b/test/integration/test_i_security.py new file mode 100644 index 00000000..67782c67 --- /dev/null +++ b/test/integration/test_i_security.py @@ -0,0 +1,630 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Security integration tests for S3 Encryption Client. + +These tests verify that the client correctly handles metadata tampering +scenarios, particularly wrapping algorithm downgrade attempts that modify +metadata to bypass encryption context validation. +""" + +import base64 +import json +import os +from datetime import datetime +from unittest.mock import MagicMock + +import boto3 +import pytest +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.primitives.padding import PKCS7 + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.decryptor import AesCbcDecryptor +from s3_encryption.exceptions import S3EncryptionClientError, S3EncryptionClientSecurityError +from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager +from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy +from s3_encryption.pipelines import GetEncryptedObjectPipeline + +bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") +region = os.environ.get("CI_AWS_REGION", "us-west-2") +kms_key_id = os.environ.get( + "CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" +) + + +def _unique_key(prefix): + return prefix + datetime.now().strftime("%Y-%m-%d-%H:%M:%S-%f") + + +def _make_client(algorithm_suite, commitment_policy, enable_legacy_wrapping=False): + """Create an S3EncryptionClient with the given config.""" + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring( + kms_client, kms_key_id, enable_legacy_wrapping_algorithms=enable_legacy_wrapping + ) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig( + keyring, + encryption_algorithm=algorithm_suite, + commitment_policy=commitment_policy, + ) + return S3EncryptionClient(wrapped_client, config) + + +class TestWrappingAlgorithmDowngradeAttack: + """Tests for wrapping algorithm downgrade scenarios. + + These tests verify behavior when the wrapping algorithm metadata is + modified from kms+context to kms. In V3 format, "kms" is not a valid + compressed wrapping algorithm code, so the client MUST reject it. + """ + + def test_v3_downgrade_wrap_alg_to_kms_rejected_without_legacy(self): + """Tampering x-amz-w from '12' to 'kms' MUST fail when legacy wrapping is disabled. + + The default KmsKeyring does not enable legacy wrapping algorithms, + so the 'kms' wrapping algorithm value should be rejected outright. + """ + key = _unique_key("sec-downgrade-no-legacy-") + data = b"sensitive data with context" + encryption_context = {"project": "alpha"} + + # 1. Encrypt normally with kms+context (V3 format) + s3ec = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) + + # 2. Attacker tampers x-amz-w from '12' to 'kms' via S3 copy + plain_s3 = boto3.client("s3") + head = plain_s3.head_object(Bucket=bucket, Key=key) + original_metadata = head["Metadata"] + assert ( + original_metadata.get("x-amz-w") == "12" + ), f"Expected x-amz-w='12', got {original_metadata.get('x-amz-w')}" + + tampered_metadata = original_metadata.copy() + tampered_metadata["x-amz-w"] = "kms" + + plain_s3.copy_object( + Bucket=bucket, + Key=key, + CopySource={"Bucket": bucket, "Key": key}, + Metadata=tampered_metadata, + MetadataDirective="REPLACE", + ) + + # 3. Decryption with mismatched context MUST fail + with pytest.raises((S3EncryptionClientError, Exception)): + s3ec.get_object(Bucket=bucket, Key=key, EncryptionContext={"project": "beta"}) + + def test_v3_downgrade_wrap_alg_to_kms_rejected_with_correct_context(self): + """Tampering x-amz-w from '12' to 'kms' MUST fail even with the original context. + + The V3 wrapping algorithm validation rejects "kms" as an invalid + compressed code regardless of what encryption context the caller + provides. The rejection happens before any context comparison. + """ + key = _unique_key("sec-downgrade-no-legacy-correct-ctx-") + data = b"sensitive data with context" + encryption_context = {"project": "alpha"} + + # 1. Encrypt normally with kms+context (V3 format) + s3ec = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) + + # 2. Tamper x-amz-w from '12' to 'kms' + plain_s3 = boto3.client("s3") + head = plain_s3.head_object(Bucket=bucket, Key=key) + tampered_metadata = head["Metadata"].copy() + tampered_metadata["x-amz-w"] = "kms" + + plain_s3.copy_object( + Bucket=bucket, + Key=key, + CopySource={"Bucket": bucket, "Key": key}, + Metadata=tampered_metadata, + MetadataDirective="REPLACE", + ) + + # 3. Decryption with the ORIGINAL (correct) context MUST still fail + with pytest.raises((S3EncryptionClientError, Exception)): + s3ec.get_object(Bucket=bucket, Key=key, EncryptionContext=encryption_context) + + def test_v3_downgrade_wrap_alg_to_kms_rejected_with_legacy(self): + """Tampering x-amz-w from '12' to 'kms' MUST still fail even with legacy enabled. + + Even when enable_legacy_wrapping_algorithms=True, the KmsV1 path + passes the *stored* encryption context to KMS Decrypt. Since the + data key was originally encrypted with the 'alpha' context, KMS + itself will reject the Decrypt call (the ciphertext is bound to + the original context). The mismatched 'beta' context should never + produce a successful decryption. + """ + key = _unique_key("sec-downgrade-legacy-") + data = b"sensitive data with context" + encryption_context = {"project": "alpha"} + + # 1. Encrypt with kms+context (V3) + s3ec_encrypt = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + s3ec_encrypt.put_object( + Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context + ) + + # 2. Attacker tampers x-amz-w from '12' to 'kms' + plain_s3 = boto3.client("s3") + head = plain_s3.head_object(Bucket=bucket, Key=key) + tampered_metadata = head["Metadata"].copy() + tampered_metadata["x-amz-w"] = "kms" + + plain_s3.copy_object( + Bucket=bucket, + Key=key, + CopySource={"Bucket": bucket, "Key": key}, + Metadata=tampered_metadata, + MetadataDirective="REPLACE", + ) + + # 3. Decrypt with legacy enabled but mismatched context MUST fail + s3ec_legacy = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + enable_legacy_wrapping=True, + ) + with pytest.raises((S3EncryptionClientError, Exception)): + s3ec_legacy.get_object(Bucket=bucket, Key=key, EncryptionContext={"project": "beta"}) + + def test_v3_downgrade_wrap_alg_correct_context_still_fails(self): + """Tampering x-amz-w from '12' to 'kms' MUST fail even with the correct context. + + The KmsV1 path uses the *stored* encryption context (from x-amz-t) + for the KMS Decrypt call. But the stored context for kms+context + includes the reserved key 'aws:x-amz-cek-alg'. When the wrapping + algorithm is changed to 'kms', the keyring may not reconstruct the + correct KMS encryption context, causing KMS to reject the call. + This verifies the attack fails regardless of what context the + caller provides. + """ + key = _unique_key("sec-downgrade-correct-ctx-") + data = b"sensitive data with context" + encryption_context = {"project": "alpha"} + + # 1. Encrypt with kms+context (V3) + s3ec = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) + + # 2. Attacker tampers x-amz-w + plain_s3 = boto3.client("s3") + head = plain_s3.head_object(Bucket=bucket, Key=key) + tampered_metadata = head["Metadata"].copy() + tampered_metadata["x-amz-w"] = "kms" + + plain_s3.copy_object( + Bucket=bucket, + Key=key, + CopySource={"Bucket": bucket, "Key": key}, + Metadata=tampered_metadata, + MetadataDirective="REPLACE", + ) + + # 3. Even with the CORRECT original context, decryption should fail + # because the wrapping algorithm mismatch corrupts the KMS call + s3ec_legacy = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + enable_legacy_wrapping=True, + ) + with pytest.raises((S3EncryptionClientError, Exception)): + s3ec_legacy.get_object(Bucket=bucket, Key=key, EncryptionContext=encryption_context) + + def test_v3_downgrade_with_matdesc_injection(self): + """Tampering x-amz-w to 'kms' AND copying x-amz-t into x-amz-m MUST be rejected. + + "kms" is not a valid V3 compressed wrapping algorithm code, so the + client rejects it before the matdesc injection has any effect. + """ + key = _unique_key("sec-v3-downgrade-matdesc-") + data = b"sensitive data with context" + encryption_context = {"project": "alpha"} + + # 1. Encrypt with kms+context (V3) + s3ec = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) + + # 2. Attacker tampers x-amz-w AND copies x-amz-t into x-amz-m + plain_s3 = boto3.client("s3") + head = plain_s3.head_object(Bucket=bucket, Key=key) + tampered_metadata = head["Metadata"].copy() + + # Downgrade wrapping algorithm + tampered_metadata["x-amz-w"] = "kms" + # Copy the original bound context from x-amz-t into x-amz-m + # so the KmsV1 path reads it as mat_desc and passes it to KMS Decrypt + tampered_metadata["x-amz-m"] = tampered_metadata["x-amz-t"] + + plain_s3.copy_object( + Bucket=bucket, + Key=key, + CopySource={"Bucket": bucket, "Key": key}, + Metadata=tampered_metadata, + MetadataDirective="REPLACE", + ) + + # 3. Decrypt with legacy enabled + mismatched context MUST fail + s3ec_legacy = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + enable_legacy_wrapping=True, + ) + with pytest.raises((S3EncryptionClientError, Exception)): + s3ec_legacy.get_object(Bucket=bucket, Key=key, EncryptionContext={"project": "beta"}) + + +class TestV2WrappingAlgorithmDowngradeAttack: + """V2 wrapping algorithm downgrade tests. + + V2 stores the wrapping algorithm in x-amz-wrap-alg. The KmsV1 ("kms") + wrapping algorithm does not support caller-provided encryption context. + When a caller provides encryption context on decrypt and the wrapping + algorithm is "kms", the client MUST reject the request. This is the + canonical behavior established by the Java AmazonS3EncryptionClientV2. + """ + + def test_v2_downgrade_wrap_alg_to_kms_correct_context(self): + """Tampering x-amz-wrap-alg to 'kms' MUST fail even with the original correct context. + + The KmsV1 wrapping algorithm does not support encryption context. + The client MUST reject when a caller provides any encryption context + and the wrapping algorithm is 'kms', regardless of whether the + context matches the stored matdesc. + """ + key = _unique_key("sec-v2-downgrade-correct-ctx-") + data = b"sensitive v2 data" + encryption_context = {"project": "alpha"} + + # 1. Encrypt with V2 format (AES_GCM, kms+context wrapping) + s3ec = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) + s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) + + # 2. Tamper x-amz-wrap-alg from 'kms+context' to 'kms' + plain_s3 = boto3.client("s3") + head = plain_s3.head_object(Bucket=bucket, Key=key) + tampered_metadata = head["Metadata"].copy() + tampered_metadata["x-amz-wrap-alg"] = "kms" + + plain_s3.copy_object( + Bucket=bucket, + Key=key, + CopySource={"Bucket": bucket, "Key": key}, + Metadata=tampered_metadata, + MetadataDirective="REPLACE", + ) + + # 3. Decrypt with legacy enabled + CORRECT original context MUST still fail + s3ec_legacy = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + enable_legacy_wrapping=True, + ) + with pytest.raises((S3EncryptionClientError, Exception)): + s3ec_legacy.get_object(Bucket=bucket, Key=key, EncryptionContext=encryption_context) + + def test_v2_downgrade_wrap_alg_to_kms_mismatched_context(self): + """Tampering x-amz-wrap-alg from 'kms+context' to 'kms' with wrong context. + + The KmsV1 wrapping algorithm does not support encryption context. + The client MUST reject when a caller provides mismatched encryption + context and the wrapping algorithm is 'kms'. + """ + key = _unique_key("sec-v2-downgrade-") + data = b"sensitive v2 data" + encryption_context = {"project": "alpha"} + + # 1. Encrypt with V2 format (AES_GCM, kms+context wrapping) + s3ec = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) + s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) + + # 2. Attacker tampers x-amz-wrap-alg from 'kms+context' to 'kms' + plain_s3 = boto3.client("s3") + head = plain_s3.head_object(Bucket=bucket, Key=key) + original_metadata = head["Metadata"] + assert ( + original_metadata.get("x-amz-wrap-alg") == "kms+context" + ), f"Expected x-amz-wrap-alg='kms+context', got {original_metadata.get('x-amz-wrap-alg')}" + + tampered_metadata = original_metadata.copy() + tampered_metadata["x-amz-wrap-alg"] = "kms" + + plain_s3.copy_object( + Bucket=bucket, + Key=key, + CopySource={"Bucket": bucket, "Key": key}, + Metadata=tampered_metadata, + MetadataDirective="REPLACE", + ) + + # 3. Decrypt with legacy enabled + mismatched context + s3ec_legacy = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + enable_legacy_wrapping=True, + ) + with pytest.raises((S3EncryptionClientError, Exception)): + s3ec_legacy.get_object(Bucket=bucket, Key=key, EncryptionContext={"project": "beta"}) + + +class TestEncryptionContextBypassAttempts: + """Tests verifying encryption context cannot be bypassed through other vectors.""" + + def test_v3_no_context_on_decrypt_after_context_on_encrypt(self): + """Omitting EncryptionContext on get_object MUST fail if object was encrypted with one.""" + key = _unique_key("sec-no-ctx-decrypt-") + data = b"data requiring context" + encryption_context = {"project": "alpha"} + + s3ec = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) + + with pytest.raises(S3EncryptionClientError): + s3ec.get_object(Bucket=bucket, Key=key) + + def test_v3_tamper_stored_context_metadata(self): + """Tampering x-amz-t (stored encryption context) MUST cause KMS Decrypt to fail. + + The KMS ciphertext is bound to the original encryption context. + Modifying x-amz-t changes what the client sends to KMS Decrypt, + causing a mismatch with the ciphertext's bound context. + """ + key = _unique_key("sec-tamper-ctx-") + data = b"data with bound context" + encryption_context = {"project": "alpha"} + + s3ec = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) + + # Tamper the stored encryption context in x-amz-t + plain_s3 = boto3.client("s3") + head = plain_s3.head_object(Bucket=bucket, Key=key) + tampered_metadata = head["Metadata"].copy() + + # Replace the stored context with attacker-controlled values + tampered_metadata["x-amz-t"] = json.dumps({"project": "beta", "aws:x-amz-cek-alg": "115"}) + + plain_s3.copy_object( + Bucket=bucket, + Key=key, + CopySource={"Bucket": bucket, "Key": key}, + Metadata=tampered_metadata, + MetadataDirective="REPLACE", + ) + + # Decryption with the tampered context should fail at KMS + with pytest.raises((S3EncryptionClientError, Exception)): + s3ec.get_object(Bucket=bucket, Key=key, EncryptionContext={"project": "beta"}) + + +class TestCBCErrorIndistinguishability: + """Tests verifying that CBC decryption errors are indistinguishable. + + A padding oracle requires the caller to distinguish between padding + errors and other decryption failures. These tests verify that all CBC + failure modes produce the same error type and message, preventing + an attacker from using error responses to deduce padding validity. + """ + + def _encrypt_cbc(self, key, iv, plaintext): + """Helper to encrypt with AES-CBC + PKCS7 padding.""" + cipher = Cipher(algorithms.AES(key), modes.CBC(iv)) + encryptor = cipher.encryptor() + padder = PKCS7(128).padder() + padded = padder.update(plaintext) + padder.finalize() + return encryptor.update(padded) + encryptor.finalize() + + def _make_cbc_decryptor(self, key, iv, content_length): + """Helper to create an AesCbcDecryptor.""" + cipher = Cipher(algorithms.AES(key), modes.CBC(iv)) + unpadder = PKCS7(128).unpadder() + return AesCbcDecryptor(cipher.decryptor(), unpadder, content_length) + + def test_wrong_key_and_tampered_ciphertext_produce_same_error(self): + """Wrong key and tampered ciphertext MUST produce identical error messages. + + Both cause PKCS7 unpadding to fail, but the error message and type + MUST be the same so an attacker cannot distinguish between them. + """ + key = os.urandom(32) + iv = os.urandom(16) + ciphertext = self._encrypt_cbc(key, iv, b"test data for padding oracle check") + + # Wrong key: decryption produces garbage, unpadding fails + wrong_key = os.urandom(32) + decryptor1 = self._make_cbc_decryptor(wrong_key, iv, len(ciphertext)) + with pytest.raises(S3EncryptionClientSecurityError) as exc1: + decryptor1.finalize(ciphertext) + + # Tampered ciphertext: last byte flipped, unpadding fails + tampered = ciphertext[:-1] + bytes([ciphertext[-1] ^ 0x01]) + decryptor2 = self._make_cbc_decryptor(key, iv, len(tampered)) + with pytest.raises(S3EncryptionClientSecurityError) as exc2: + decryptor2.finalize(tampered) + + # Both MUST produce the same error message + assert str(exc1.value) == str(exc2.value), ( + f"Error messages differ: wrong_key={str(exc1.value)!r}, " + f"tampered={str(exc2.value)!r}" + ) + + # Neither message should contain details about the underlying failure + assert ( + "padding" not in str(exc1.value).lower() + ), f"Error message leaks padding information: {str(exc1.value)!r}" + + def test_truncated_ciphertext_produces_same_error(self): + """Truncated ciphertext MUST produce the same error as padding failure. + + A non-block-aligned ciphertext causes a different exception in the + cryptography library. The error message MUST be identical to prevent + an attacker from distinguishing truncation from padding failure. + """ + key = os.urandom(32) + iv = os.urandom(16) + ciphertext = self._encrypt_cbc(key, iv, b"test data for truncation check") + + # Padding failure (wrong key) + wrong_key = os.urandom(32) + decryptor1 = self._make_cbc_decryptor(wrong_key, iv, len(ciphertext)) + with pytest.raises(S3EncryptionClientSecurityError) as exc1: + decryptor1.finalize(ciphertext) + + # Truncated ciphertext (not block-aligned) + truncated = ciphertext[:-3] + decryptor2 = self._make_cbc_decryptor(key, iv, len(truncated)) + with pytest.raises(S3EncryptionClientSecurityError) as exc2: + decryptor2.finalize(truncated) + + # Both MUST produce the same error message + assert str(exc1.value) == str(exc2.value), ( + f"Error messages differ: padding_fail={str(exc1.value)!r}, " + f"truncated={str(exc2.value)!r}" + ) + + +class TestInstructionFileFormatConfusion: + """Tests for instruction file metadata injection causing format confusion. + + When a V3 object uses instruction files, the instruction file metadata + is merged with object metadata. If an attacker injects V2-format keys + into the instruction file (or directly into object metadata), the merged + metadata may contain keys from multiple format versions. The client + detects this via has_exclusive_key_collision() and the V2+V3 content + key coexistence check, rejecting the tampered metadata before format + dispatch. + """ + + def test_v2_keys_injected_into_v3_metadata_rejected(self): + """Injecting V2 keys into V3 object metadata MUST be rejected. + + Encrypt a V3 object, then tamper the S3 metadata to add V2 keys + alongside the existing V3 content keys. The client MUST reject + this because V2 and V3 keys should never coexist. + """ + key = _unique_key("sec-v2-inject-v3-") + data = b"data for format confusion test" + encryption_context = {"project": "alpha"} + + # 1. Encrypt with V3 format + s3ec = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) + + # 2. Tamper: inject V2 keys alongside existing V3 metadata + plain_s3 = boto3.client("s3") + head = plain_s3.head_object(Bucket=bucket, Key=key) + tampered_metadata = head["Metadata"].copy() + + # Add V2 keys — the V3 keys (x-amz-c, x-amz-d, x-amz-i, x-amz-3, x-amz-w) remain + tampered_metadata["x-amz-key-v2"] = tampered_metadata.get("x-amz-3", "fake") + tampered_metadata["x-amz-cek-alg"] = "AES/GCM/NoPadding" + tampered_metadata["x-amz-iv"] = "AAAAAAAAAAAAAAAA" + tampered_metadata["x-amz-wrap-alg"] = "kms+context" + + plain_s3.copy_object( + Bucket=bucket, + Key=key, + CopySource={"Bucket": bucket, "Key": key}, + Metadata=tampered_metadata, + MetadataDirective="REPLACE", + ) + + # 3. Decrypt MUST fail — metadata has both V2 and V3 keys + s3ec_legacy = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + enable_legacy_wrapping=True, + ) + with pytest.raises((S3EncryptionClientError, Exception)): + s3ec_legacy.get_object(Bucket=bucket, Key=key, EncryptionContext=encryption_context) + + def test_exclusive_key_collision_detected_during_decrypt(self): + """The decrypt pipeline MUST reject metadata with exclusive key collisions. + + When merged metadata contains both V2 and V3 exclusive keys, + the pipeline detects the collision and raises an error. + """ + # Create a mock CMM that would return decryption materials + mock_cmm = MagicMock(spec=DefaultCryptoMaterialsManager) + + pipeline = GetEncryptedObjectPipeline( + cmm=mock_cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + enable_legacy_unauthenticated_modes=False, + ) + + # Build a response with merged V2+V3 metadata (simulating the + # instruction file injection after merge) + fake_edk = base64.b64encode(os.urandom(32)).decode() + fake_iv = base64.b64encode(os.urandom(12)).decode() + fake_message_id = base64.b64encode(os.urandom(28)).decode() + fake_commitment = base64.b64encode(os.urandom(28)).decode() + + merged_metadata = { + # V2 keys (from attacker instruction file) + "x-amz-key-v2": fake_edk, + "x-amz-cek-alg": "AES/GCM/NoPadding", + "x-amz-iv": fake_iv, + "x-amz-wrap-alg": "kms+context", + "x-amz-matdesc": '{"aws:x-amz-cek-alg": "AES/GCM/NoPadding"}', + # V3 keys (from object metadata) + "x-amz-c": "115", + "x-amz-d": fake_commitment, + "x-amz-i": fake_message_id, + "x-amz-w": "12", + "x-amz-3": fake_edk, + } + + fake_body = MagicMock() + fake_body.read.return_value = os.urandom(48) # fake ciphertext + + response = { + "Body": fake_body, + "Metadata": merged_metadata, + "ContentLength": 48, + } + + # This SHOULD raise an error due to exclusive key collision, + # but currently routes to _decrypt_v2 instead + with pytest.raises(S3EncryptionClientError): + pipeline.decrypt( + response, + instruction_suffix=".instruction", + enable_delayed_authentication=False, + encryption_context={}, + ) diff --git a/test/test_kms_keyring.py b/test/test_kms_keyring.py index 0a5d66de..8c5b2ab2 100644 --- a/test/test_kms_keyring.py +++ b/test/test_kms_keyring.py @@ -467,3 +467,58 @@ def test_on_decrypt_fails_when_kms_fails(self): keyring.on_decrypt(dec_materials) assert exc_info.value is kms_exception + + def test_on_decrypt_kms_v1_rejects_any_encryption_context(self): + """KmsV1 path must reject any caller-provided encryption context.""" + mock_kms_client = MagicMock() + keyring = KmsKeyring( + mock_kms_client, + "arn:aws:kms:us-east-1:123456789012:key/test-key", + enable_legacy_wrapping_algorithms=True, + ) + + edk = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms", + encrypted_data_key=b"encrypted-key", + ) + dec_materials = DecryptionMaterials( + iv=b"initialization-vector", + encrypted_data_keys=[edk], + encryption_context_stored={"project": "alpha", "kms_cmk_id": "some-key"}, + encryption_context_from_request={"project": "alpha"}, + ) + + with pytest.raises(S3EncryptionClientError, match="not supported with the KmsV1"): + keyring.on_decrypt(dec_materials) + + mock_kms_client.decrypt.assert_not_called() + + def test_on_decrypt_kms_v1_rejects_mismatched_encryption_context(self): + """KmsV1 path must reject mismatched caller-provided encryption context.""" + mock_kms_client = MagicMock() + keyring = KmsKeyring( + mock_kms_client, + "arn:aws:kms:us-east-1:123456789012:key/test-key", + enable_legacy_wrapping_algorithms=True, + ) + + edk = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms", + encrypted_data_key=b"encrypted-key", + ) + dec_materials = DecryptionMaterials( + iv=b"initialization-vector", + encrypted_data_keys=[edk], + encryption_context_stored={"project": "alpha", "kms_cmk_id": "some-key"}, + encryption_context_from_request={"project": "beta"}, + ) + + with pytest.raises(S3EncryptionClientError, match="not supported with the KmsV1"): + keyring.on_decrypt(dec_materials) + + mock_kms_client.decrypt.assert_not_called() + + # KMS should never be called when context doesn't match + mock_kms_client.decrypt.assert_not_called() diff --git a/test/test_pipelines.py b/test/test_pipelines.py index a66880e4..edd9ba8d 100644 --- a/test/test_pipelines.py +++ b/test/test_pipelines.py @@ -434,3 +434,38 @@ def test_decrypt_instruction_file_empty_metadata_raises(self): bucket="test-bucket", key="test-key", ) + + def test_decrypt_rejects_exclusive_key_collision(self): + """Metadata with both V2 and V3 EDK keys MUST be rejected.""" + import base64 + import os + + mock_cmm = Mock() + pipeline = GetEncryptedObjectPipeline( + cmm=mock_cmm, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) + + fake_edk = base64.b64encode(os.urandom(32)).decode() + fake_iv = base64.b64encode(os.urandom(12)).decode() + # Metadata with both V2 (x-amz-key-v2) and V3 (x-amz-3) EDK keys + metadata = { + "x-amz-key-v2": fake_edk, + "x-amz-cek-alg": "AES/GCM/NoPadding", + "x-amz-iv": fake_iv, + "x-amz-wrap-alg": "kms+context", + "x-amz-3": fake_edk, + "x-amz-c": "115", + "x-amz-w": "12", + "x-amz-d": base64.b64encode(os.urandom(28)).decode(), + "x-amz-i": base64.b64encode(os.urandom(28)).decode(), + } + + mock_response = { + "Body": BytesIO(os.urandom(48)), + "Metadata": metadata, + "ContentLength": 48, + } + + with pytest.raises(S3EncryptionClientError, match="multiple format versions"): + pipeline.decrypt(mock_response, ".instruction", enable_delayed_authentication=False) From 6bbfd0ad8fd756f56387a75ff2f07b0031b88c0d Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Mon, 27 Apr 2026 13:27:47 -0700 Subject: [PATCH 69/81] feat: add instruction file config to provide a way to disable instruction files (#176) --- src/s3_encryption/__init__.py | 26 +- src/s3_encryption/instruction_file_config.py | 34 +++ src/s3_encryption/pipelines.py | 10 + .../test_i_s3_encryption_instruction_file.py | 249 +++++++++++++++++ test/test_instruction_file_config.py | 254 ++++++++++++++++++ test/test_s3_encryption_client_delete.py | 19 ++ ...est_s3_encryption_client_delete_objects.py | 26 ++ 7 files changed, 610 insertions(+), 8 deletions(-) create mode 100644 src/s3_encryption/instruction_file_config.py create mode 100644 test/test_instruction_file_config.py diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index f684c6a9..a8ab239b 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -11,6 +11,7 @@ from .exceptions import S3EncryptionClientError from .instruction_file import parse_instruction_file +from .instruction_file_config import InstructionFileConfig from .materials.crypto_materials_manager import ( AbstractCryptoMaterialsManager, DefaultCryptoMaterialsManager, @@ -57,6 +58,9 @@ class S3EncryptionClientConfig: before GCM tag verification. Defaults to False. Has no effect for CBC encrypted ciphertext, which is always streamed as there is no authentication tag. + instruction_file_config: Configuration for instruction file behavior. + Defaults to InstructionFileConfig() which enables instruction file + reads on GetObject. Raises: S3EncryptionClientError: If the encryption algorithm is legacy, or if @@ -86,6 +90,8 @@ class S3EncryptionClientConfig: ##% Delayed Authentication mode MUST be set to false by default. enable_delayed_authentication: bool = field(default=False) + instruction_file_config: InstructionFileConfig = field(factory=InstructionFileConfig) + @cmm.default def _default_cmm_for_keyring(self): return DefaultCryptoMaterialsManager(self.keyring) @@ -248,6 +254,7 @@ def on_get_object_after_call(self, parsed, **kwargs): commitment_policy=self.config.commitment_policy, s3_client=getattr(self._context, _CTX_S3_CLIENT, None), enable_legacy_unauthenticated_modes=self.config.enable_legacy_unauthenticated_modes, + instruction_file_config=self.config.instruction_file_config, ) decrypted_data = pipeline.decrypt( response, @@ -428,8 +435,9 @@ def delete_object(self, **kwargs): ##= type=implementation ##% - DeleteObject MUST delete the associated instruction file ##% using the default instruction file suffix. - instruction_key = kwargs["Key"] + instruction_file_suffix - self.wrapped_s3_client.delete_object(Bucket=kwargs["Bucket"], Key=instruction_key) + if not self.config.instruction_file_config.disable_delete_object: + instruction_key = kwargs["Key"] + instruction_file_suffix + self.wrapped_s3_client.delete_object(Bucket=kwargs["Bucket"], Key=instruction_key) return response except S3EncryptionClientError: @@ -469,12 +477,14 @@ def delete_objects(self, **kwargs): ##= type=implementation ##% - DeleteObjects MUST delete each of the corresponding instruction files ##% using the default instruction file suffix. - instruction_objects = [ - {"Key": obj["Key"] + instruction_file_suffix} for obj in kwargs["Delete"]["Objects"] - ] - self.wrapped_s3_client.delete_objects( - Bucket=kwargs["Bucket"], Delete={"Objects": instruction_objects} - ) + if not self.config.instruction_file_config.disable_delete_objects: + instruction_objects = [ + {"Key": obj["Key"] + instruction_file_suffix} + for obj in kwargs["Delete"]["Objects"] + ] + self.wrapped_s3_client.delete_objects( + Bucket=kwargs["Bucket"], Delete={"Objects": instruction_objects} + ) return response except S3EncryptionClientError: diff --git a/src/s3_encryption/instruction_file_config.py b/src/s3_encryption/instruction_file_config.py new file mode 100644 index 00000000..73320533 --- /dev/null +++ b/src/s3_encryption/instruction_file_config.py @@ -0,0 +1,34 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Instruction file configuration for S3 Encryption Client. + +This module provides configuration for instruction file behavior +during encryption and decryption operations. +""" + +from attrs import define, field + + +@define +class InstructionFileConfig: + """Configuration for instruction file behavior in the S3 Encryption Client. + + Controls whether the client will interact with instruction files + as part of GetObject, DeleteObject, and DeleteObjects operations. + + Attributes: + disable_get_object: If True, the client will not attempt to fetch + instruction files during GetObject (decryption) and will raise + an error if the object's metadata implies an instruction file + is required. Defaults to False. + disable_delete_object: If True, the client will not attempt to + delete the associated instruction file during DeleteObject. + Defaults to False. + disable_delete_objects: If True, the client will not attempt to + delete the associated instruction files during DeleteObjects. + Defaults to False. + """ + + disable_get_object: bool = field(default=False) + disable_delete_object: bool = field(default=False) + disable_delete_objects: bool = field(default=False) diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index c4fe4867..15255173 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -20,6 +20,7 @@ from .decryptor import AesCbcDecryptor, AesGcmDecryptor from .exceptions import S3EncryptionClientError from .instruction_file import fetch_instruction_file +from .instruction_file_config import InstructionFileConfig from .key_derivation import derive_keys, verify_commitment from .materials.crypto_materials_manager import AbstractCryptoMaterialsManager from .materials.encrypted_data_key import EncryptedDataKey @@ -181,6 +182,7 @@ class GetEncryptedObjectPipeline: commitment_policy: CommitmentPolicy = field() s3_client: object = field(default=None) enable_legacy_unauthenticated_modes: bool = field(default=False) + instruction_file_config: InstructionFileConfig = field(factory=InstructionFileConfig) # Map content cipher metadata values to AlgorithmSuite _CONTENT_CIPHER_TO_ALGORITHM_SUITE = { @@ -258,6 +260,14 @@ def decrypt( # Check if we need to fetch instruction file if metadata.should_use_instruction_file(): + if self.instruction_file_config.disable_get_object: + raise S3EncryptionClientError( + "Exception encountered while fetching Instruction File. " + "Ensure the object you are attempting to decrypt has been encrypted " + "using the S3 Encryption Client and instruction files are enabled. " + f"bucket: {bucket}\n key: {key}" + ) + if self.s3_client is None: raise S3EncryptionClientError("s3_client required to fetch instruction file") if bucket is None or key is None: diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index b2d2b5f4..c46176f5 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -291,3 +291,252 @@ def test_decrypt_large_v3_instruction_file_delayed_auth(): total += len(chunk) assert total == LARGE_FILE_SIZE + + +# --- InstructionFileConfig integration tests --- + + +def test_instruction_file_config_disabled_raises_on_instruction_file_object(): + """When instruction file get is disabled, decrypting an instruction-file object MUST fail.""" + from s3_encryption.exceptions import S3EncryptionClientError + from s3_encryption.instruction_file_config import InstructionFileConfig + + key = TEST_OBJECTS["v3_instruction_file"] + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig( + keyring, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + instruction_file_config=InstructionFileConfig(disable_get_object=True), + ) + s3ec = S3EncryptionClient(wrapped_client, config) + + with pytest.raises( + S3EncryptionClientError, match="Exception encountered while fetching Instruction File" + ): + s3ec.get_object(Bucket=bucket, Key=key) + + +def test_instruction_file_config_enabled_still_decrypts(): + """When instruction file get is explicitly enabled, decryption MUST succeed as before.""" + from s3_encryption.instruction_file_config import InstructionFileConfig + + key = TEST_OBJECTS["v3_instruction_file"] + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig( + keyring, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + instruction_file_config=InstructionFileConfig(disable_get_object=False), + ) + s3ec = S3EncryptionClient(wrapped_client, config) + + response = s3ec.get_object(Bucket=bucket, Key=key) + output = response["Body"].read().decode("utf-8") + + assert output == "static-v3-instruction-file-from-java-v4" + + +def test_instruction_file_config_disabled_allows_non_instruction_file_objects(): + """When instruction file get is disabled, objects with metadata in headers MUST still decrypt.""" + from s3_encryption.instruction_file_config import InstructionFileConfig + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + + # First, put an object using default config (metadata in object headers) + put_config = S3EncryptionClientConfig(keyring) + put_client = S3EncryptionClient(boto3.client("s3"), put_config) + + test_key = f"instruction-file-config-test-{uuid.uuid4()}" + plaintext = b"hello from instruction file config test" + put_client.put_object(Bucket=bucket, Key=test_key, Body=plaintext) + + try: + # Now decrypt with instruction file get disabled + config = S3EncryptionClientConfig( + keyring, + instruction_file_config=InstructionFileConfig(disable_get_object=True), + ) + s3ec = S3EncryptionClient(wrapped_client, config) + + response = s3ec.get_object(Bucket=bucket, Key=test_key) + output = response["Body"].read() + + assert output == plaintext + finally: + wrapped_client.delete_object(Bucket=bucket, Key=test_key) + + +def test_instruction_file_config_default_still_decrypts_instruction_files(): + """Default InstructionFileConfig (no explicit config) MUST still decrypt instruction files.""" + key = TEST_OBJECTS["v3_instruction_file"] + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + # No instruction_file_config specified — should use default (enabled) + config = S3EncryptionClientConfig( + keyring, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + s3ec = S3EncryptionClient(wrapped_client, config) + + response = s3ec.get_object(Bucket=bucket, Key=key) + output = response["Body"].read().decode("utf-8") + + assert output == "static-v3-instruction-file-from-java-v4" + + +# --- InstructionFileConfig delete_object / delete_objects integration tests --- + + +def _object_exists(bucket_name, key_name): + """Return True if the object exists in the bucket.""" + from botocore.exceptions import ClientError + + s3 = boto3.client("s3") + try: + s3.head_object(Bucket=bucket_name, Key=key_name) + return True + except ClientError as e: + if e.response["Error"]["Code"] == "404": + return False + raise + + +def test_delete_object_skips_instruction_file_when_disabled(): + """delete_object with disable_delete_object=True must NOT delete the instruction file.""" + from s3_encryption.instruction_file_config import InstructionFileConfig + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + plain_s3 = boto3.client("s3") + + test_key = f"ifc-delete-obj-skip-{uuid.uuid4()}" + instr_key = test_key + ".instruction" + + # Put an encrypted object and a fake instruction file + default_client = S3EncryptionClient(boto3.client("s3"), S3EncryptionClientConfig(keyring)) + default_client.put_object(Bucket=bucket, Key=test_key, Body=b"data") + plain_s3.put_object(Bucket=bucket, Key=instr_key, Body=b"{}") + + try: + # Delete with instruction file deletion disabled + config = S3EncryptionClientConfig( + keyring, + instruction_file_config=InstructionFileConfig(disable_delete_object=True), + ) + s3ec = S3EncryptionClient(boto3.client("s3"), config) + s3ec.delete_object(Bucket=bucket, Key=test_key) + + # Object should be gone, instruction file should remain + assert not _object_exists(bucket, test_key) + assert _object_exists(bucket, instr_key) + finally: + # Clean up the instruction file + plain_s3.delete_object(Bucket=bucket, Key=instr_key) + + +def test_delete_object_deletes_instruction_file_when_not_disabled(): + """delete_object with default config must delete the instruction file.""" + from s3_encryption.instruction_file_config import InstructionFileConfig + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + plain_s3 = boto3.client("s3") + + test_key = f"ifc-delete-obj-default-{uuid.uuid4()}" + instr_key = test_key + ".instruction" + + default_client = S3EncryptionClient(boto3.client("s3"), S3EncryptionClientConfig(keyring)) + default_client.put_object(Bucket=bucket, Key=test_key, Body=b"data") + plain_s3.put_object(Bucket=bucket, Key=instr_key, Body=b"{}") + + # Delete with default config (instruction file deletion enabled) + config = S3EncryptionClientConfig( + keyring, + instruction_file_config=InstructionFileConfig(disable_delete_object=False), + ) + s3ec = S3EncryptionClient(boto3.client("s3"), config) + s3ec.delete_object(Bucket=bucket, Key=test_key) + + assert not _object_exists(bucket, test_key) + assert not _object_exists(bucket, instr_key) + + +def test_delete_objects_skips_instruction_files_when_disabled(): + """delete_objects with disable_delete_objects=True must NOT delete instruction files.""" + from s3_encryption.instruction_file_config import InstructionFileConfig + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + plain_s3 = boto3.client("s3") + + keys = [f"ifc-delete-objs-skip-{uuid.uuid4()}" for _ in range(2)] + instr_keys = [k + ".instruction" for k in keys] + + default_client = S3EncryptionClient(boto3.client("s3"), S3EncryptionClientConfig(keyring)) + for key in keys: + default_client.put_object(Bucket=bucket, Key=key, Body=b"data") + for instr_key in instr_keys: + plain_s3.put_object(Bucket=bucket, Key=instr_key, Body=b"{}") + + try: + config = S3EncryptionClientConfig( + keyring, + instruction_file_config=InstructionFileConfig(disable_delete_objects=True), + ) + s3ec = S3EncryptionClient(boto3.client("s3"), config) + s3ec.delete_objects( + Bucket=bucket, + Delete={"Objects": [{"Key": k} for k in keys]}, + ) + + for key in keys: + assert not _object_exists(bucket, key) + for instr_key in instr_keys: + assert _object_exists(bucket, instr_key) + finally: + plain_s3.delete_objects( + Bucket=bucket, + Delete={"Objects": [{"Key": k} for k in instr_keys]}, + ) + + +def test_delete_objects_deletes_instruction_files_when_not_disabled(): + """delete_objects with default config must delete instruction files.""" + from s3_encryption.instruction_file_config import InstructionFileConfig + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + plain_s3 = boto3.client("s3") + + keys = [f"ifc-delete-objs-default-{uuid.uuid4()}" for _ in range(2)] + instr_keys = [k + ".instruction" for k in keys] + + default_client = S3EncryptionClient(boto3.client("s3"), S3EncryptionClientConfig(keyring)) + for key in keys: + default_client.put_object(Bucket=bucket, Key=key, Body=b"data") + for instr_key in instr_keys: + plain_s3.put_object(Bucket=bucket, Key=instr_key, Body=b"{}") + + config = S3EncryptionClientConfig( + keyring, + instruction_file_config=InstructionFileConfig(disable_delete_objects=False), + ) + s3ec = S3EncryptionClient(boto3.client("s3"), config) + s3ec.delete_objects( + Bucket=bucket, + Delete={"Objects": [{"Key": k} for k in keys]}, + ) + + for key in keys: + assert not _object_exists(bucket, key) + for instr_key in instr_keys: + assert not _object_exists(bucket, instr_key) diff --git a/test/test_instruction_file_config.py b/test/test_instruction_file_config.py new file mode 100644 index 00000000..851fc605 --- /dev/null +++ b/test/test_instruction_file_config.py @@ -0,0 +1,254 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for InstructionFileConfig and its integration with S3EncryptionClientConfig.""" + +import base64 +import json +import os +from io import BytesIO +from unittest.mock import Mock + +import pytest + +from s3_encryption import S3EncryptionClientConfig +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.instruction_file_config import InstructionFileConfig +from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager +from s3_encryption.materials.keyring import S3Keyring +from s3_encryption.materials.materials import CommitmentPolicy +from s3_encryption.pipelines import GetEncryptedObjectPipeline + + +class TestInstructionFileConfig: + """Tests for the InstructionFileConfig attrs class.""" + + def test_defaults_all_false(self): + """All disable flags default to False.""" + config = InstructionFileConfig() + assert config.disable_get_object is False + assert config.disable_delete_object is False + assert config.disable_delete_objects is False + + def test_disable_get_object(self): + """disable_get_object can be set to True.""" + config = InstructionFileConfig(disable_get_object=True) + assert config.disable_get_object is True + assert config.disable_delete_object is False + assert config.disable_delete_objects is False + + def test_disable_delete_object(self): + """disable_delete_object can be set independently.""" + config = InstructionFileConfig(disable_delete_object=True) + assert config.disable_get_object is False + assert config.disable_delete_object is True + assert config.disable_delete_objects is False + + def test_disable_delete_objects(self): + """disable_delete_objects can be set independently.""" + config = InstructionFileConfig(disable_delete_objects=True) + assert config.disable_get_object is False + assert config.disable_delete_object is False + assert config.disable_delete_objects is True + + def test_all_disabled(self): + """All flags can be set to True simultaneously.""" + config = InstructionFileConfig( + disable_get_object=True, + disable_delete_object=True, + disable_delete_objects=True, + ) + assert config.disable_get_object is True + assert config.disable_delete_object is True + assert config.disable_delete_objects is True + + +class TestS3EncryptionClientConfigInstructionFileConfig: + """Tests for instruction_file_config on S3EncryptionClientConfig.""" + + def test_default_instruction_file_config(self): + """S3EncryptionClientConfig defaults to InstructionFileConfig with all enabled.""" + mock_keyring = Mock(spec=S3Keyring) + config = S3EncryptionClientConfig(keyring=mock_keyring) + assert isinstance(config.instruction_file_config, InstructionFileConfig) + assert config.instruction_file_config.disable_get_object is False + + def test_custom_instruction_file_config(self): + """S3EncryptionClientConfig accepts a custom InstructionFileConfig.""" + mock_keyring = Mock(spec=S3Keyring) + ifc = InstructionFileConfig(disable_get_object=True) + config = S3EncryptionClientConfig(keyring=mock_keyring, instruction_file_config=ifc) + assert config.instruction_file_config.disable_get_object is True + + def test_instruction_file_config_does_not_affect_other_config(self): + """Setting instruction_file_config does not change other defaults.""" + mock_keyring = Mock(spec=S3Keyring) + ifc = InstructionFileConfig(disable_get_object=True) + config = S3EncryptionClientConfig(keyring=mock_keyring, instruction_file_config=ifc) + assert config.enable_delayed_authentication is False + assert config.enable_legacy_unauthenticated_modes is False + + +class TestPipelineInstructionFileGetDisabled: + """Tests for GetEncryptedObjectPipeline when instruction file get is disabled.""" + + def test_decrypt_raises_when_instruction_file_disabled_and_needed(self): + """Pipeline MUST raise when instruction file is needed but disabled.""" + object_metadata = {} + + mock_keyring = Mock(spec=S3Keyring) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + mock_s3_client = Mock() + + pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + s3_client=mock_s3_client, + instruction_file_config=InstructionFileConfig(disable_get_object=True), + ) + + mock_response = { + "Body": BytesIO(b"encrypted-data"), + "Metadata": object_metadata, + } + + with pytest.raises( + S3EncryptionClientError, + match="Exception encountered while fetching Instruction File", + ): + pipeline.decrypt( + mock_response, + instruction_suffix=".instruction", + enable_delayed_authentication=False, + bucket="test-bucket", + key="test-key", + ) + + mock_s3_client.get_object.assert_not_called() + + def test_decrypt_raises_when_instruction_file_disabled_v3_partial_metadata(self): + """Pipeline MUST raise when V3 object has partial metadata requiring instruction file.""" + object_metadata = { + "x-amz-c": "115", + "x-amz-d": base64.b64encode(b"key-commitment-data").decode("utf-8"), + "x-amz-i": base64.b64encode(b"test-message-id").decode("utf-8"), + } + + mock_keyring = Mock(spec=S3Keyring) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + mock_s3_client = Mock() + + pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + s3_client=mock_s3_client, + instruction_file_config=InstructionFileConfig(disable_get_object=True), + ) + + mock_response = { + "Body": BytesIO(b"encrypted-data"), + "Metadata": object_metadata, + } + + with pytest.raises( + S3EncryptionClientError, + match="Exception encountered while fetching Instruction File", + ): + pipeline.decrypt( + mock_response, + instruction_suffix=".instruction", + enable_delayed_authentication=False, + bucket="test-bucket", + key="test-key", + ) + + mock_s3_client.get_object.assert_not_called() + + def test_decrypt_succeeds_when_instruction_file_disabled_but_not_needed(self): + """Objects with metadata in headers decrypt fine regardless of config.""" + object_metadata = { + "x-amz-iv": base64.b64encode(os.urandom(12)).decode("utf-8"), + "x-amz-key-v2": base64.b64encode(b"encrypted-key-data").decode("utf-8"), + "x-amz-wrap-alg": "kms+context", + "x-amz-matdesc": json.dumps({"kms_cmk_id": "test-key-id"}), + "x-amz-cek-alg": "AES/GCM/NoPadding", + "x-amz-tag-len": "128", + } + + mock_keyring = Mock(spec=S3Keyring) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + + pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + s3_client=None, + instruction_file_config=InstructionFileConfig(disable_get_object=True), + ) + + mock_response = { + "Body": BytesIO(b"encrypted-data"), + "Metadata": object_metadata, + "ContentLength": 100, + } + + mock_keyring.on_decrypt.side_effect = Exception("Keyring called — no instruction file") + + with pytest.raises(Exception, match="Keyring called"): + pipeline.decrypt( + mock_response, + instruction_suffix=".instruction", + enable_delayed_authentication=False, + bucket="test-bucket", + key="test-key", + ) + + def test_decrypt_fetches_instruction_file_when_not_disabled(self): + """Pipeline fetches instruction file normally when disable_get_object is False.""" + object_metadata = {} + + instruction_file_metadata = { + "x-amz-iv": base64.b64encode(os.urandom(12)).decode("utf-8"), + "x-amz-key-v2": base64.b64encode(b"encrypted-key-data").decode("utf-8"), + "x-amz-wrap-alg": "kms+context", + "x-amz-matdesc": json.dumps({"kms_cmk_id": "test-key-id"}), + "x-amz-cek-alg": "AES/GCM/NoPadding", + "x-amz-tag-len": "128", + "x-amz-crypto-instr-file": "", + } + + mock_s3_client = Mock() + mock_s3_client.get_object.return_value = { + "Body": BytesIO(b""), + "Metadata": instruction_file_metadata, + } + + mock_keyring = Mock(spec=S3Keyring) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + + pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + s3_client=mock_s3_client, + instruction_file_config=InstructionFileConfig(disable_get_object=False), + ) + + mock_response = { + "Body": BytesIO(b"encrypted-test-data"), + "Metadata": object_metadata, + } + + mock_keyring.on_decrypt.side_effect = Exception( + "Keyring called - instruction file was fetched" + ) + + with pytest.raises(Exception, match="Keyring called"): + pipeline.decrypt( + mock_response, + instruction_suffix=".instruction", + enable_delayed_authentication=False, + bucket="test-bucket", + key="test-key", + ) + + mock_s3_client.get_object.assert_called_once_with( + Bucket="test-bucket", Key="test-key.instruction" + ) diff --git a/test/test_s3_encryption_client_delete.py b/test/test_s3_encryption_client_delete.py index 1279abab..897ccf3f 100644 --- a/test/test_s3_encryption_client_delete.py +++ b/test/test_s3_encryption_client_delete.py @@ -106,3 +106,22 @@ def test_instruction_file_suffix_not_forwarded_to_s3(self): # First call (object delete) should not contain InstructionFileSuffix assert mock_s3.delete_object.call_args_list[0] == call(Bucket="bucket", Key="key") + + def test_instruction_file_not_deleted_when_disabled(self): + """delete_object skips instruction file deletion when disable_delete_object is True.""" + from s3_encryption.instruction_file_config import InstructionFileConfig + + mock_keyring = Mock(spec=S3Keyring) + mock_s3 = Mock() + mock_s3.meta.events = Mock() + config = S3EncryptionClientConfig( + keyring=mock_keyring, + instruction_file_config=InstructionFileConfig(disable_delete_object=True), + ) + s3ec = S3EncryptionClient(wrapped_s3_client=mock_s3, config=config) + + s3ec.delete_object(Bucket="bucket", Key="key") + + # Only one call — the object itself, no instruction file delete + assert mock_s3.delete_object.call_count == 1 + assert mock_s3.delete_object.call_args_list[0] == call(Bucket="bucket", Key="key") diff --git a/test/test_s3_encryption_client_delete_objects.py b/test/test_s3_encryption_client_delete_objects.py index c1045ca3..4c4b99b5 100644 --- a/test/test_s3_encryption_client_delete_objects.py +++ b/test/test_s3_encryption_client_delete_objects.py @@ -186,3 +186,29 @@ def test_preserves_version_ids_in_objects(self): ], }, ) + + def test_instruction_files_not_deleted_when_disabled(self): + """delete_objects skips instruction file deletion when disable_delete_objects is True.""" + from s3_encryption.instruction_file_config import InstructionFileConfig + + mock_keyring = Mock(spec=S3Keyring) + mock_s3 = Mock() + mock_s3.meta.events = Mock() + mock_s3.delete_objects.return_value = {"Deleted": [{"Key": "key1"}, {"Key": "key2"}]} + config = S3EncryptionClientConfig( + keyring=mock_keyring, + instruction_file_config=InstructionFileConfig(disable_delete_objects=True), + ) + s3ec = S3EncryptionClient(wrapped_s3_client=mock_s3, config=config) + + s3ec.delete_objects( + Bucket="bucket", + Delete={"Objects": [{"Key": "key1"}, {"Key": "key2"}]}, + ) + + # Only one call — the objects themselves, no instruction file delete + assert mock_s3.delete_objects.call_count == 1 + assert mock_s3.delete_objects.call_args_list[0] == call( + Bucket="bucket", + Delete={"Objects": [{"Key": "key1"}, {"Key": "key2"}]}, + ) From 732100eb0078ccbc759f4888164706b92cb63d1a Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Thu, 7 May 2026 12:56:06 -0700 Subject: [PATCH 70/81] chore: guard against None dict, always default to {} (#177) * chore: guard against None dict, always default to {} --- src/s3_encryption/__init__.py | 8 ++-- src/s3_encryption/_utils.py | 12 +++++ src/s3_encryption/instruction_file.py | 3 +- .../materials/crypto_materials_manager.py | 3 +- src/s3_encryption/materials/materials.py | 9 ++-- src/s3_encryption/pipelines.py | 3 +- test/test_decryption_materials.py | 17 +++++++ test/test_encryption_materials.py | 16 +++++++ test/test_encryption_materials_integration.py | 16 +++++++ test/test_exceptions.py | 21 +++++++++ test/test_pipelines.py | 45 +++++++++++++++++++ 11 files changed, 142 insertions(+), 11 deletions(-) create mode 100644 src/s3_encryption/_utils.py diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index a8ab239b..6ee409ad 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -9,6 +9,7 @@ from botocore.exceptions import ClientError from botocore.response import StreamingBody +from ._utils import safe_get_dict from .exceptions import S3EncryptionClientError from .instruction_file import parse_instruction_file from .instruction_file_config import InstructionFileConfig @@ -198,7 +199,7 @@ def on_put_object_before_call(self, params, **kwargs): params["body"] = encrypted_data - headers = params.get("headers", {}) + headers = safe_get_dict(params, "headers") # Add encryption metadata to headers if encryption_metadata: @@ -244,7 +245,7 @@ def on_get_object_after_call(self, parsed, **kwargs): # Create a response dict that matches what the pipeline expects response = { "Body": parsed.get("Body"), - "Metadata": parsed.get("Metadata", {}), + "Metadata": safe_get_dict(parsed, "Metadata"), "ContentLength": content_length, } @@ -286,8 +287,7 @@ def process_instruction_file(self, parsed): ) # In plaintext mode, parse instruction file and append to metadata - # Metadata may be present but None, so `or {}` handles that case - existing_metadata = parsed.get("Metadata", {}) or {} + existing_metadata = safe_get_dict(parsed, "Metadata") instruction_data = body.read() instruction_metadata = parse_instruction_file(instruction_data, instruction_key) diff --git a/src/s3_encryption/_utils.py b/src/s3_encryption/_utils.py new file mode 100644 index 00000000..4997b973 --- /dev/null +++ b/src/s3_encryption/_utils.py @@ -0,0 +1,12 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Internal utility helpers for the S3 Encryption Client.""" + + +def safe_get_dict(source: dict, key: str) -> dict: + """Get a dict value from *source*, defaulting to {} if missing or None. + + This avoids the common pitfall where ``d.get(key, {})`` returns None + when the key exists but its value is explicitly None. + """ + return source.get(key, {}) or {} diff --git a/src/s3_encryption/instruction_file.py b/src/s3_encryption/instruction_file.py index 60305d17..61f9b167 100644 --- a/src/s3_encryption/instruction_file.py +++ b/src/s3_encryption/instruction_file.py @@ -11,6 +11,7 @@ from botocore.exceptions import ClientError +from ._utils import safe_get_dict from .exceptions import S3EncryptionClientError from .metadata import VALID_S3EC_METADATA_KEYS @@ -109,7 +110,7 @@ def fetch_instruction_file(s3_client, bucket: str, key: str) -> dict[str, Any]: s3_client._s3ec_plugin_context.instruction_file_mode = False # In plaintext mode, the event handler places parsed metadata in Metadata field - metadata = response.get("Metadata", {}) + metadata = safe_get_dict(response, "Metadata") # Verify metadata is not empty if not metadata: diff --git a/src/s3_encryption/materials/crypto_materials_manager.py b/src/s3_encryption/materials/crypto_materials_manager.py index 82eab454..6a7dd3e8 100644 --- a/src/s3_encryption/materials/crypto_materials_manager.py +++ b/src/s3_encryption/materials/crypto_materials_manager.py @@ -10,6 +10,7 @@ from attrs import define +from .._utils import safe_get_dict from .keyring import AbstractKeyring from .materials import DecryptionMaterials, EncryptionMaterials @@ -74,7 +75,7 @@ def get_encryption_materials(self, enc_mats_request): # Convert dictionary to EncryptionMaterials if needed if isinstance(enc_mats_request, dict): materials = EncryptionMaterials( - encryption_context=enc_mats_request.get("encryption_context", {}) + encryption_context=safe_get_dict(enc_mats_request, "encryption_context") ) else: materials = enc_mats_request diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index 80f682f0..4f91330f 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -12,6 +12,7 @@ from attrs import define, field +from .._utils import safe_get_dict from .encrypted_data_key import EncryptedDataKey @@ -232,7 +233,7 @@ def from_dict(cls, materials_dict: dict[str, Any]) -> "EncryptionMaterials": EncryptionMaterials: A new instance with fields populated from the dictionary """ return cls( - encryption_context=materials_dict.get("encryption_context", {}), + encryption_context=safe_get_dict(materials_dict, "encryption_context"), encrypted_data_key=materials_dict.get("encrypted_data_key"), plaintext_data_key=materials_dict.get("plaintext_data_key"), ) @@ -292,9 +293,9 @@ def from_dict(cls, materials_dict: dict[str, Any]) -> "DecryptionMaterials": return cls( iv=materials_dict.get("iv"), encrypted_data_keys=materials_dict.get("encrypted_data_keys", []), - encryption_context_stored=materials_dict.get("encryption_context_stored", {}), - encryption_context_from_request=materials_dict.get( - "encryption_context_from_request", {} + encryption_context_stored=safe_get_dict(materials_dict, "encryption_context_stored"), + encryption_context_from_request=safe_get_dict( + materials_dict, "encryption_context_from_request" ), plaintext_data_key=materials_dict.get("plaintext_data_key"), ) diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 15255173..2b1fe061 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -16,6 +16,7 @@ from cryptography.hazmat.primitives.ciphers.aead import AESGCM from cryptography.hazmat.primitives.padding import PKCS7 +from ._utils import safe_get_dict from .buffered_decrypt import one_shot_decrypt from .decryptor import AesCbcDecryptor, AesGcmDecryptor from .exceptions import S3EncryptionClientError @@ -250,7 +251,7 @@ def decrypt( # Convert the metadata dictionary to an ObjectMetadata instance streaming_body: StreamingBody = response.get("Body") content_length = response.get("ContentLength") - encryption_metadata = response.get("Metadata", {}) + encryption_metadata = safe_get_dict(response, "Metadata") metadata = ObjectMetadata.from_dict(encryption_metadata) # Use empty dict if encryption_context is None diff --git a/test/test_decryption_materials.py b/test/test_decryption_materials.py index c160b509..6dd51df6 100644 --- a/test/test_decryption_materials.py +++ b/test/test_decryption_materials.py @@ -85,3 +85,20 @@ def test_to_dict(self): assert materials_dict["encryption_context_stored"] == {"key1": "value1"} assert materials_dict["encryption_context_from_request"] == {"key2": "value2"} assert materials_dict["plaintext_data_key"] == b"plaintext-data-key" + + def test_from_dict_with_none_encryption_contexts(self): + """DecryptionMaterials.from_dict should handle None encryption contexts.""" + materials_dict = { + "encryption_context_stored": None, + "encryption_context_from_request": None, + } + materials = DecryptionMaterials.from_dict(materials_dict) + assert materials.encryption_context_stored == {} + assert materials.encryption_context_from_request == {} + + def test_from_dict_with_missing_encryption_contexts(self): + """DecryptionMaterials.from_dict should default to {} when context keys are missing.""" + materials_dict = {} + materials = DecryptionMaterials.from_dict(materials_dict) + assert materials.encryption_context_stored == {} + assert materials.encryption_context_from_request == {} diff --git a/test/test_encryption_materials.py b/test/test_encryption_materials.py index 54d80146..943a3c13 100644 --- a/test/test_encryption_materials.py +++ b/test/test_encryption_materials.py @@ -53,3 +53,19 @@ def test_to_dict(self): assert materials_dict["encryption_context"] == {"key1": "value1"} assert materials_dict["encrypted_data_key"] == edk assert materials_dict["plaintext_data_key"] == b"plaintext-data-key" + + def test_from_dict_with_none_encryption_context(self): + """EncryptionMaterials.from_dict should handle None encryption_context.""" + materials_dict = { + "encryption_context": None, + "encrypted_data_key": None, + "plaintext_data_key": None, + } + materials = EncryptionMaterials.from_dict(materials_dict) + assert materials.encryption_context == {} + + def test_from_dict_with_missing_encryption_context(self): + """EncryptionMaterials.from_dict should default to {} when key is missing.""" + materials_dict = {} + materials = EncryptionMaterials.from_dict(materials_dict) + assert materials.encryption_context == {} diff --git a/test/test_encryption_materials_integration.py b/test/test_encryption_materials_integration.py index e9e59023..a02343a1 100644 --- a/test/test_encryption_materials_integration.py +++ b/test/test_encryption_materials_integration.py @@ -90,3 +90,19 @@ def test_cmm_get_encryption_materials_with_materials(self): assert result.encryption_context == {"key1": "value1"} assert result.encrypted_data_key is not None assert result.plaintext_data_key is not None + + def test_cmm_get_encryption_materials_with_none_encryption_context(self): + """DefaultCryptoMaterialsManager handles None encryption_context in dict request.""" + keyring = MagicMock() + keyring.on_encrypt.return_value = EncryptionMaterials( + encryption_context={}, + plaintext_data_key=b"key", + ) + cmm = DefaultCryptoMaterialsManager(keyring=keyring) + + # Pass a dict with None encryption_context — should not raise TypeError + cmm.get_encryption_materials({"encryption_context": None}) + + # Keyring should receive empty dict, not None + call_args = keyring.on_encrypt.call_args[0][0] + assert call_args.encryption_context == {} diff --git a/test/test_exceptions.py b/test/test_exceptions.py index f93e3d9d..4fe46bcc 100644 --- a/test/test_exceptions.py +++ b/test/test_exceptions.py @@ -49,3 +49,24 @@ def test_inherits_from_botocore_error(self): def test_can_be_caught_as_botocore_error(self): with pytest.raises(BotoCoreError): raise S3EncryptionClientSecurityError("test security error") + + +from s3_encryption._utils import safe_get_dict + + +class TestSafeGetDict: + def test_returns_value_when_present(self): + assert safe_get_dict({"key": {"a": 1}}, "key") == {"a": 1} + + def test_returns_empty_dict_when_key_missing(self): + assert safe_get_dict({}, "key") == {} + + def test_returns_empty_dict_when_value_is_none(self): + assert safe_get_dict({"key": None}, "key") == {} + + def test_returns_empty_dict_for_empty_value(self): + assert safe_get_dict({"key": {}}, "key") == {} + + def test_preserves_non_empty_dict(self): + data = {"x": "y", "z": "w"} + assert safe_get_dict({"meta": data}, "meta") == data diff --git a/test/test_pipelines.py b/test/test_pipelines.py index edd9ba8d..e06542f0 100644 --- a/test/test_pipelines.py +++ b/test/test_pipelines.py @@ -469,3 +469,48 @@ def test_decrypt_rejects_exclusive_key_collision(self): with pytest.raises(S3EncryptionClientError, match="multiple format versions"): pipeline.decrypt(mock_response, ".instruction", enable_delayed_authentication=False) + + +class TestGetEncryptedObjectPipelineNoneMetadata: + """Tests that None Metadata in response is handled gracefully.""" + + def test_decrypt_with_none_metadata(self): + """Pipeline should not raise TypeError when Metadata is None.""" + mock_cmm = Mock() + pipeline = GetEncryptedObjectPipeline( + cmm=mock_cmm, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) + + response = { + "Body": BytesIO(b"test"), + "ContentLength": 4, + "Metadata": None, + } + + with pytest.raises(S3EncryptionClientError): + pipeline.decrypt( + response, + instruction_suffix=".instruction", + enable_delayed_authentication=False, + ) + + def test_decrypt_with_missing_metadata(self): + """Pipeline should not raise TypeError when Metadata key is absent.""" + mock_cmm = Mock() + pipeline = GetEncryptedObjectPipeline( + cmm=mock_cmm, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) + + response = { + "Body": BytesIO(b"test"), + "ContentLength": 4, + } + + with pytest.raises(S3EncryptionClientError): + pipeline.decrypt( + response, + instruction_suffix=".instruction", + enable_delayed_authentication=False, + ) From 08b6eb4387cc51ac26b68d2c64b378471c4c6e7a Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Fri, 8 May 2026 09:26:42 -0700 Subject: [PATCH 71/81] chore: disable ranged gets (#178) --- src/s3_encryption/__init__.py | 6 +++ test/integration/test_i_ranged_get.py | 72 +++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 test/integration/test_i_ranged_get.py diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 6ee409ad..caad4f66 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -511,6 +511,12 @@ def get_object(self, **kwargs): Raises: S3EncryptionClientError: If decryption fails or the object is not properly encrypted. """ + # Ranged gets are not supported — decryption requires the full ciphertext. + if "Range" in kwargs: + raise S3EncryptionClientError( + "Ranged gets are currently not supported by the S3 Encryption Client for Python." + ) + # Extract EncryptionContext if provided (not a standard S3 parameter) encryption_context = kwargs.pop("EncryptionContext", None) _validate_encryption_context(encryption_context) diff --git a/test/integration/test_i_ranged_get.py b/test/integration/test_i_ranged_get.py new file mode 100644 index 00000000..d9bb65af --- /dev/null +++ b/test/integration/test_i_ranged_get.py @@ -0,0 +1,72 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Integration test for ranged get (Range parameter on get_object). + +The S3 Encryption Client does not support ranged gets because decryption +requires the full ciphertext (IV, encrypted data, and auth tag). Passing +a Range parameter retrieves only a slice of the ciphertext, which causes +decryption to fail. +""" + +import os +from datetime import datetime + +import boto3 +import pytest + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy + +bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") +region = os.environ.get("CI_AWS_REGION", "us-west-2") +kms_key_id = os.environ.get( + "CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" +) + + +def _make_client(algorithm_suite, commitment_policy): + """Create an S3EncryptionClient with the given algorithm config.""" + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig( + keyring, + encryption_algorithm=algorithm_suite, + commitment_policy=commitment_policy, + ) + return S3EncryptionClient(wrapped_client, config) + + +def _unique_key(prefix): + """Generate a unique S3 key with a timestamp suffix.""" + return prefix + datetime.now().strftime("%Y-%m-%d-%H:%M:%S-%f") + + +ALGORITHM_CONFIGS = [ + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + id="AES_GCM", + ), + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + id="KC_GCM", + ), +] + + +@pytest.mark.parametrize(("algorithm_suite", "commitment_policy"), ALGORITHM_CONFIGS) +def test_ranged_get_fails(algorithm_suite, commitment_policy): + """Ranged gets are rejected with a clear error.""" + key = _unique_key("ranged-get-") + data = b"A" * 1024 + + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + + # Attempt a ranged get — should raise immediately with a clear message + with pytest.raises(S3EncryptionClientError, match="Ranged gets are currently not supported"): + s3ec.get_object(Bucket=bucket, Key=key, Range="bytes=0-255") From ab05b8c1618be33a4829c0adb35b61025bf5dd65 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Mon, 11 May 2026 10:59:43 -0700 Subject: [PATCH 72/81] chore: reduce perf rounds (#180) --- .github/workflows/python-perf.yml | 2 +- test/performance/conftest.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-perf.yml b/.github/workflows/python-perf.yml index c5a4ab41..8b4bed00 100644 --- a/.github/workflows/python-perf.yml +++ b/.github/workflows/python-perf.yml @@ -52,7 +52,7 @@ jobs: env: CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} - PERF_NUM_ROUNDS: "30" + PERF_NUM_ROUNDS: "10" - name: Generate performance HTML report if: always() diff --git a/test/performance/conftest.py b/test/performance/conftest.py index 2e686a30..aaf9c934 100644 --- a/test/performance/conftest.py +++ b/test/performance/conftest.py @@ -17,8 +17,8 @@ ) # Performance test configuration -NUM_ROUNDS = int(os.environ.get("PERF_NUM_ROUNDS", "30")) -OBJECT_SIZES_MB = [10, 25, 50] +NUM_ROUNDS = int(os.environ.get("PERF_NUM_ROUNDS", "10")) +OBJECT_SIZES_MB = [10, 50] def _make_s3ec(algorithm_suite, commitment_policy): From c3167c99115304441e895672549b54004e78bcf8 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Mon, 11 May 2026 11:34:46 -0700 Subject: [PATCH 73/81] chore: bump deps (#181) * pin major version of runtime deps --- pyproject.toml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 781e89e0..4e898108 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,20 +9,20 @@ license = {text = "Apache-2.0"} readme = "README.md" requires-python = ">=3.11" dependencies = [ - "boto3>=1.37.2", - "cryptography>=45.0.6", - "attrs>=25.1.0", + "boto3>=1.43.6,<2", + "cryptography>=48.0.0,<49", + "attrs>=26.1.0,<27", ] [project.optional-dependencies] test = [ - "pytest>=8.4.1", - "pytest-cov>=6.1.1", + "pytest>=9.0.3", + "pytest-cov>=7.1.0", ] dev = [ - "black>=24.3.0,<27.0.0", - "ruff>=0.3.0", - "boto3-stubs~=1.42.49", + "black>=26.3.1", + "ruff>=0.15.12", + "boto3-stubs~=1.43.6", ] [build-system] From b4bfc00f22282c413fa70157389d02bb792c7d49 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Mon, 11 May 2026 12:25:29 -0700 Subject: [PATCH 74/81] chore: move this project to V4 (#179) * chore: move this project to V4 * start from 3 --- .github/workflows/python-integ.yml | 2 +- pyproject.toml | 2 +- src/s3_encryption/__init__.py | 2 +- test-server/README.md | 4 ++-- ...eyCommitmentPolicyEncryptFailureTests.java | 8 ++++---- .../amazon/encryption/s3/TestUtils.java | 20 +++++++++---------- .../.duvet/.gitignore | 0 .../.duvet/config.toml | 0 .../.gitignore | 0 .../Makefile | 6 +++--- .../README.md | 0 .../poetry.lock | 0 .../pyproject.toml | 0 .../src/__init__.py | 0 .../src/main.py | 0 .../tests/__init__.py | 0 16 files changed, 22 insertions(+), 22 deletions(-) rename test-server/{python-v3-server => python-v4-server}/.duvet/.gitignore (100%) rename test-server/{python-v3-server => python-v4-server}/.duvet/config.toml (100%) rename test-server/{python-v3-server => python-v4-server}/.gitignore (100%) rename test-server/{python-v3-server => python-v4-server}/Makefile (89%) rename test-server/{python-v3-server => python-v4-server}/README.md (100%) rename test-server/{python-v3-server => python-v4-server}/poetry.lock (100%) rename test-server/{python-v3-server => python-v4-server}/pyproject.toml (100%) rename test-server/{python-v3-server => python-v4-server}/src/__init__.py (100%) rename test-server/{python-v3-server => python-v4-server}/src/main.py (100%) rename test-server/{python-v3-server => python-v4-server}/tests/__init__.py (100%) diff --git a/.github/workflows/python-integ.yml b/.github/workflows/python-integ.yml index ec50a72c..e2d710cc 100644 --- a/.github/workflows/python-integ.yml +++ b/.github/workflows/python-integ.yml @@ -31,7 +31,7 @@ jobs: uses: actions/cache@v5 with: path: ~/.cache/uv - key: ${{ runner.os }}-uv-${{ hashFiles('./test-server/python-v3-server/**/pyproject.toml') }} + key: ${{ runner.os }}-uv-${{ hashFiles('./test-server/python-v4-server/**/pyproject.toml') }} restore-keys: | ${{ runner.os }}-uv- diff --git a/pyproject.toml b/pyproject.toml index 4e898108..5e94ee4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "amazon-s3-encryption-client-python" -version = "0.1.0" +version = "3.0.0" description = "This library provides an S3 client that supports client-side encryption." authors = [ {name = "AWS Crypto Tools", email = "aws-crypto-tools@amazon.com"} diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index caad4f66..ad07fe43 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -1,6 +1,6 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Top-level S3 Encryption Client v3 for Python package.""" +"""Top-level S3 Encryption Client v4 for Python package.""" import io import threading diff --git a/test-server/README.md b/test-server/README.md index 48187fc3..68796f30 100644 --- a/test-server/README.md +++ b/test-server/README.md @@ -28,8 +28,8 @@ make ci # Start Python and Java servers in parallel make start-servers -# Start only the Python S3EC V3 server -make start-python-v3-server +# Start only the Python S3EC V4 server +make start-python-v4-server # Run Java tests make run-tests diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java index 46df7de1..f705f89d 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java @@ -31,7 +31,7 @@ * with the commitment policy is rejected by the S3EC — either at client creation * or at PutObject time. * - * Currently scoped to Python V3 only. Other languages can be enabled by + * Currently scoped to Python V4 only. Other languages can be enabled by * switching the MethodSource to a broader provider (e.g. improvedClientsForTest). */ @DisplayName("Key Commitment Policy — Encrypt Failures") @@ -42,7 +42,7 @@ public class KeyCommitmentPolicyEncryptFailureTests { .build(); @ParameterizedTest(name = "{0}: REQUIRE_ENCRYPT_ALLOW_DECRYPT with non-committing GCM MUST fail to encrypt") - @MethodSource("software.amazon.encryption.s3.TestUtils#pythonV3ClientForTest") + @MethodSource("software.amazon.encryption.s3.TestUtils#pythonV4ClientForTest") void require_encrypt_allow_decrypt_with_non_committing_gcm_must_fail( TestUtils.LanguageServerTarget language ) { @@ -58,7 +58,7 @@ void require_encrypt_allow_decrypt_with_non_committing_gcm_must_fail( } @ParameterizedTest(name = "{0}: REQUIRE_ENCRYPT_REQUIRE_DECRYPT with non-committing GCM MUST fail to encrypt") - @MethodSource("software.amazon.encryption.s3.TestUtils#pythonV3ClientForTest") + @MethodSource("software.amazon.encryption.s3.TestUtils#pythonV4ClientForTest") void require_encrypt_require_decrypt_with_non_committing_gcm_must_fail( TestUtils.LanguageServerTarget language ) { @@ -74,7 +74,7 @@ void require_encrypt_require_decrypt_with_non_committing_gcm_must_fail( } @ParameterizedTest(name = "{0}: FORBID_ENCRYPT_ALLOW_DECRYPT with committing GCM MUST fail to encrypt") - @MethodSource("software.amazon.encryption.s3.TestUtils#pythonV3ClientForTest") + @MethodSource("software.amazon.encryption.s3.TestUtils#pythonV4ClientForTest") void forbid_encrypt_allow_decrypt_with_committing_gcm_must_fail( TestUtils.LanguageServerTarget language ) { diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index c1464eaf..5f4ce9d6 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -65,8 +65,8 @@ public class TestUtils { public static final String JAVA_V3_TRANSITION = "Java-V3-Transition"; public static final String JAVA_V4 = "Java-V4"; - // No Python S3EC versions are released. Only test V3 as the "vN+1" version. - public static final String PYTHON_V3 = "Python-V3"; + // No Python S3EC versions are released. Only test V4 as the "vN+1" version. + public static final String PYTHON_V4 = "Python-V4"; public static final String GO_V3_TRANSITION = "Go-V3-Transition"; public static final String GO_V4 = "Go-V4"; @@ -101,7 +101,7 @@ public class TestUtils { // Languages that reject caller-provided encryption context when the // wrapping algorithm is KmsV1 ("kms"). public static final Set KMSV1_ENCRYPTION_CONTEXT_UNSUPPORTED = - Set.of(PYTHON_V3); + Set.of(PYTHON_V4); public static final Set RE_ENCRYPT_SUPPORTED = Set.of(JAVA_V3_TRANSITION, JAVA_V4); @@ -131,11 +131,11 @@ public class TestUtils { // Go does not write with instruction files public static final Set INSTRUCTION_FILE_PUT_UNSUPPORTED = - Set.of(GO_V3_TRANSITION, GO_V4, PYTHON_V3); + Set.of(GO_V3_TRANSITION, GO_V4, PYTHON_V4); // Not implemented yet in Python. public static final Set INSTRUCTION_FILE_GET_UNSUPPORTED = - Set.of(PYTHON_V3); + Set.of(PYTHON_V4); // Languages that support custom instruction file suffix on GetObject // Only Java, Ruby, and PHP servers have been updated with this feature @@ -163,7 +163,7 @@ public class TestUtils { public static final Set IMPROVED_VERSIONS = Set.of( JAVA_V4, - PYTHON_V3, + PYTHON_V4, GO_V4, NET_V4, CPP_V3, @@ -175,7 +175,7 @@ public class TestUtils { static { final Map servers = new LinkedHashMap<>(); - servers.put(PYTHON_V3, new LanguageServerTarget(PYTHON_V3, "8081")); + servers.put(PYTHON_V4, new LanguageServerTarget(PYTHON_V4, "8081")); servers.put(CPP_V2_TRANSITION, new LanguageServerTarget(CPP_V2_TRANSITION, "8097")); servers.put(CPP_V3, new LanguageServerTarget(CPP_V3, "8091")); servers.put(GO_V4, new LanguageServerTarget(GO_V4, "8089")); @@ -367,13 +367,13 @@ public static Stream improvedClientsForTest() { } /** - * Get stream of arguments for the Python V3 client only. + * Get stream of arguments for the Python V4 client only. * Other languages can be added to this set as their commitment policy * validation is confirmed. */ - public static Stream pythonV3ClientForTest() { + public static Stream pythonV4ClientForTest() { return serverMap.values().stream() - .filter(target -> PYTHON_V3.equals(target.getLanguageName())) + .filter(target -> PYTHON_V4.equals(target.getLanguageName())) .map(Arguments::of); } diff --git a/test-server/python-v3-server/.duvet/.gitignore b/test-server/python-v4-server/.duvet/.gitignore similarity index 100% rename from test-server/python-v3-server/.duvet/.gitignore rename to test-server/python-v4-server/.duvet/.gitignore diff --git a/test-server/python-v3-server/.duvet/config.toml b/test-server/python-v4-server/.duvet/config.toml similarity index 100% rename from test-server/python-v3-server/.duvet/config.toml rename to test-server/python-v4-server/.duvet/config.toml diff --git a/test-server/python-v3-server/.gitignore b/test-server/python-v4-server/.gitignore similarity index 100% rename from test-server/python-v3-server/.gitignore rename to test-server/python-v4-server/.gitignore diff --git a/test-server/python-v3-server/Makefile b/test-server/python-v4-server/Makefile similarity index 89% rename from test-server/python-v3-server/Makefile rename to test-server/python-v4-server/Makefile index 930c950c..063aa372 100644 --- a/test-server/python-v3-server/Makefile +++ b/test-server/python-v4-server/Makefile @@ -6,20 +6,20 @@ PID_FILE := server.pid PORT := 8081 build-server: - @echo "Building Python V3 server..." + @echo "Building Python V4 server..." python -m venv .venv .venv/bin/python -m ensurepip .venv/bin/python -m pip install -e . .venv/bin/python -m pip install -e ../.. start-server: - @echo "Starting Python V3 server..." + @echo "Starting Python V4 server..." AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ .venv/bin/python src/main.py > server.log 2>&1 & echo $$! > $(PID_FILE) - @echo "Python V3 server starting..." + @echo "Python V4 server starting..." stop-server: @echo "Stopping server on port $(PORT)..." diff --git a/test-server/python-v3-server/README.md b/test-server/python-v4-server/README.md similarity index 100% rename from test-server/python-v3-server/README.md rename to test-server/python-v4-server/README.md diff --git a/test-server/python-v3-server/poetry.lock b/test-server/python-v4-server/poetry.lock similarity index 100% rename from test-server/python-v3-server/poetry.lock rename to test-server/python-v4-server/poetry.lock diff --git a/test-server/python-v3-server/pyproject.toml b/test-server/python-v4-server/pyproject.toml similarity index 100% rename from test-server/python-v3-server/pyproject.toml rename to test-server/python-v4-server/pyproject.toml diff --git a/test-server/python-v3-server/src/__init__.py b/test-server/python-v4-server/src/__init__.py similarity index 100% rename from test-server/python-v3-server/src/__init__.py rename to test-server/python-v4-server/src/__init__.py diff --git a/test-server/python-v3-server/src/main.py b/test-server/python-v4-server/src/main.py similarity index 100% rename from test-server/python-v3-server/src/main.py rename to test-server/python-v4-server/src/main.py diff --git a/test-server/python-v3-server/tests/__init__.py b/test-server/python-v4-server/tests/__init__.py similarity index 100% rename from test-server/python-v3-server/tests/__init__.py rename to test-server/python-v4-server/tests/__init__.py From 0f336a53632eb494ceef1050795a7e24b78415bc Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Wed, 13 May 2026 09:34:31 -0700 Subject: [PATCH 75/81] chore: apply user agent string with version (#182) --- src/s3_encryption/__init__.py | 4 +- src/s3_encryption/_utils.py | 12 ++++++ src/s3_encryption/materials/kms_keyring.py | 5 +++ test/test_user_agent.py | 43 ++++++++++++++++++++++ 4 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 test/test_user_agent.py diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index ad07fe43..3f188510 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -9,7 +9,7 @@ from botocore.exceptions import ClientError from botocore.response import StreamingBody -from ._utils import safe_get_dict +from ._utils import _USER_AGENT_SUFFIX, append_user_agent, safe_get_dict from .exceptions import S3EncryptionClientError from .instruction_file import parse_instruction_file from .instruction_file_config import InstructionFileConfig @@ -350,6 +350,8 @@ def __attrs_post_init__(self): # Expose plugin context on wrapped client for instruction file fetching self.wrapped_s3_client._s3ec_plugin_context = self._plugin._context + append_user_agent(self.wrapped_s3_client, _USER_AGENT_SUFFIX) + # Register event handlers using boto3's event system event_system = self.wrapped_s3_client.meta.events event_system.register("before-call.s3.PutObject", self._plugin.on_put_object_before_call) diff --git a/src/s3_encryption/_utils.py b/src/s3_encryption/_utils.py index 4997b973..7cab9e27 100644 --- a/src/s3_encryption/_utils.py +++ b/src/s3_encryption/_utils.py @@ -2,6 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 """Internal utility helpers for the S3 Encryption Client.""" +import importlib.metadata + +_PACKAGE_VERSION = importlib.metadata.version("amazon-s3-encryption-client-python") +_USER_AGENT_SUFFIX = f"S3ECPy/{_PACKAGE_VERSION}" + def safe_get_dict(source: dict, key: str) -> dict: """Get a dict value from *source*, defaulting to {} if missing or None. @@ -10,3 +15,10 @@ def safe_get_dict(source: dict, key: str) -> dict: when the key exists but its value is explicitly None. """ return source.get(key, {}) or {} + + +def append_user_agent(client, suffix: str): + """Append a suffix to the User-Agent header of a boto3 client.""" + existing = client.meta.config.user_agent_extra or "" + sep = " " if existing else "" + client.meta.config.user_agent_extra = f"{existing}{sep}{suffix}" diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index abd6fad4..6550a38b 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -53,6 +53,11 @@ class KmsKeyring(S3Keyring): ##% The KmsV1 mode MUST be only enabled when legacy wrapping algorithms are enabled. enable_legacy_wrapping_algorithms: bool = field(default=False) + def __attrs_post_init__(self): # noqa: D105 + from .._utils import _USER_AGENT_SUFFIX, append_user_agent + + append_user_agent(self.kms_client, _USER_AGENT_SUFFIX) + def on_encrypt(self, enc_materials): """Process encryption materials using KMS. diff --git a/test/test_user_agent.py b/test/test_user_agent.py new file mode 100644 index 00000000..ad7f6a30 --- /dev/null +++ b/test/test_user_agent.py @@ -0,0 +1,43 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for user agent string injection.""" + +import boto3 + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption._utils import _PACKAGE_VERSION, _USER_AGENT_SUFFIX +from s3_encryption.materials.kms_keyring import KmsKeyring + + +class TestUserAgent: + def test_user_agent_suffix_format(self): + assert f"S3ECPy/{_PACKAGE_VERSION}" == _USER_AGENT_SUFFIX + + def test_s3_client_gets_user_agent(self): + s3 = boto3.client("s3", region_name="us-east-1") + kms = boto3.client("kms", region_name="us-east-1") + keyring = KmsKeyring(kms, "arn:aws:kms:us-east-1:000000000000:key/fake") + config = S3EncryptionClientConfig(keyring=keyring) + + S3EncryptionClient(s3, config) + + assert _USER_AGENT_SUFFIX in s3.meta.config.user_agent_extra + + def test_kms_client_gets_user_agent(self): + kms = boto3.client("kms", region_name="us-east-1") + KmsKeyring(kms, "arn:aws:kms:us-east-1:000000000000:key/fake") + + assert _USER_AGENT_SUFFIX in kms.meta.config.user_agent_extra + + def test_existing_user_agent_extra_preserved(self): + s3 = boto3.client("s3", region_name="us-east-1") + s3.meta.config.user_agent_extra = "existing-agent/1.0" + + kms = boto3.client("kms", region_name="us-east-1") + keyring = KmsKeyring(kms, "arn:aws:kms:us-east-1:000000000000:key/fake") + config = S3EncryptionClientConfig(keyring=keyring) + + S3EncryptionClient(s3, config) + + assert "existing-agent/1.0" in s3.meta.config.user_agent_extra + assert _USER_AGENT_SUFFIX in s3.meta.config.user_agent_extra From c4ab1c03d2c8fa3414c04d0e3d881670389347b8 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Wed, 13 May 2026 14:30:57 -0700 Subject: [PATCH 76/81] chore: remove Black, use ruff for formatting (#184) --- .github/workflows/lint.yml | 1 + Makefile | 10 ++++++---- pyproject.toml | 6 ------ src/s3_encryption/instruction_file.py | 2 +- src/s3_encryption/materials/kms_keyring.py | 6 ++---- src/s3_encryption/pipelines.py | 5 +---- test/integration/test_i_security.py | 21 ++++++++++----------- test/performance/generate_report.py | 22 +++++++++++----------- test/test_stream.py | 4 ---- 9 files changed, 32 insertions(+), 45 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b374a9a7..a1ef5e2d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -25,4 +25,5 @@ jobs: - name: Install dependencies and run linting run: | make install + make format-check make lint diff --git a/Makefile b/Makefile index e764a496..d01d75a3 100644 --- a/Makefile +++ b/Makefile @@ -10,14 +10,16 @@ install: # Run linting checks lint: - uv run black --check src/ test/ - # Enforce ruff checks on src/ but allow test/ to fail uv run ruff check src/ uv run ruff check test/ || true -# Format code with Black and Ruff +# Check formatting (no changes, just verify) +format-check: + uv run ruff format --check src/ test/ + +# Format code format: - uv run black src/ test/ + uv run ruff format src/ test/ uv run ruff check --fix src/ test/ # Run all tests with combined coverage diff --git a/pyproject.toml b/pyproject.toml index 5e94ee4d..7fb8a58e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,6 @@ test = [ "pytest-cov>=7.1.0", ] dev = [ - "black>=26.3.1", "ruff>=0.15.12", "boto3-stubs~=1.43.6", ] @@ -32,11 +31,6 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["src/s3_encryption"] -[tool.black] -line-length = 100 -target-version = ["py311"] -include = '\.pyi?$' - [tool.ruff] line-length = 100 target-version = "py311" diff --git a/src/s3_encryption/instruction_file.py b/src/s3_encryption/instruction_file.py index 61f9b167..61df766f 100644 --- a/src/s3_encryption/instruction_file.py +++ b/src/s3_encryption/instruction_file.py @@ -47,7 +47,7 @@ def parse_instruction_file(instruction_data: bytes, key: str) -> dict[str, Any]: # Validate that it's a dictionary if not isinstance(metadata, dict): raise S3EncryptionClientError( - f"Instruction file must contain a JSON object, " f"got {type(metadata).__name__}: {key}" + f"Instruction file must contain a JSON object, got {type(metadata).__name__}: {key}" ) # Validate that all keys are S3EC metadata keys diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index 6550a38b..083cca63 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -169,8 +169,7 @@ def on_decrypt(self, dec_materials, encrypted_data_keys=None): ##% context. if KMS_CONTEXT_DEFAULT_KEY in encryption_context_from_request: raise S3EncryptionClientError( - f"{KMS_CONTEXT_DEFAULT_KEY} is a reserved key for the " - f"S3 encryption client" + f"{KMS_CONTEXT_DEFAULT_KEY} is a reserved key for the S3 encryption client" ) ##= specification/s3-encryption/materials/s3-kms-keyring.md#kms-context @@ -188,8 +187,7 @@ def on_decrypt(self, dec_materials, encrypted_data_keys=None): if encryption_context_stored_copy != encryption_context_from_request: # TODO: modeled error raise S3EncryptionClientError( - "Provided encryption context does not match information " - "retrieved from S3" + "Provided encryption context does not match information retrieved from S3" ) ##= specification/s3-encryption/materials/s3-kms-keyring.md#decryptdatakey diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 2b1fe061..23a169ae 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -260,7 +260,6 @@ def decrypt( # Check if we need to fetch instruction file if metadata.should_use_instruction_file(): - if self.instruction_file_config.disable_get_object: raise S3EncryptionClientError( "Exception encountered while fetching Instruction File. " @@ -371,9 +370,7 @@ def decrypt( ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is NOT enabled, ##% the S3EC MUST throw an error which details that client was ##% not configured to decrypt objects with ALG_AES_256_CBC_IV16_NO_KDF. - if ( - algorithm_suite.is_legacy and not self.enable_legacy_unauthenticated_modes - ): # noqa: SIM102 + if algorithm_suite.is_legacy and not self.enable_legacy_unauthenticated_modes: # noqa: SIM102 ##= specification/s3-encryption/decryption.md#legacy-decryption ##= type=implementation ##% The S3EC MUST NOT decrypt objects encrypted using legacy unauthenticated algorithm suites diff --git a/test/integration/test_i_security.py b/test/integration/test_i_security.py index 67782c67..3d871e1f 100644 --- a/test/integration/test_i_security.py +++ b/test/integration/test_i_security.py @@ -81,9 +81,9 @@ def test_v3_downgrade_wrap_alg_to_kms_rejected_without_legacy(self): plain_s3 = boto3.client("s3") head = plain_s3.head_object(Bucket=bucket, Key=key) original_metadata = head["Metadata"] - assert ( - original_metadata.get("x-amz-w") == "12" - ), f"Expected x-amz-w='12', got {original_metadata.get('x-amz-w')}" + assert original_metadata.get("x-amz-w") == "12", ( + f"Expected x-amz-w='12', got {original_metadata.get('x-amz-w')}" + ) tampered_metadata = original_metadata.copy() tampered_metadata["x-amz-w"] = "kms" @@ -348,9 +348,9 @@ def test_v2_downgrade_wrap_alg_to_kms_mismatched_context(self): plain_s3 = boto3.client("s3") head = plain_s3.head_object(Bucket=bucket, Key=key) original_metadata = head["Metadata"] - assert ( - original_metadata.get("x-amz-wrap-alg") == "kms+context" - ), f"Expected x-amz-wrap-alg='kms+context', got {original_metadata.get('x-amz-wrap-alg')}" + assert original_metadata.get("x-amz-wrap-alg") == "kms+context", ( + f"Expected x-amz-wrap-alg='kms+context', got {original_metadata.get('x-amz-wrap-alg')}" + ) tampered_metadata = original_metadata.copy() tampered_metadata["x-amz-wrap-alg"] = "kms" @@ -476,14 +476,13 @@ def test_wrong_key_and_tampered_ciphertext_produce_same_error(self): # Both MUST produce the same error message assert str(exc1.value) == str(exc2.value), ( - f"Error messages differ: wrong_key={str(exc1.value)!r}, " - f"tampered={str(exc2.value)!r}" + f"Error messages differ: wrong_key={str(exc1.value)!r}, tampered={str(exc2.value)!r}" ) # Neither message should contain details about the underlying failure - assert ( - "padding" not in str(exc1.value).lower() - ), f"Error message leaks padding information: {str(exc1.value)!r}" + assert "padding" not in str(exc1.value).lower(), ( + f"Error message leaks padding information: {str(exc1.value)!r}" + ) def test_truncated_ciphertext_produces_same_error(self): """Truncated ciphertext MUST produce the same error as padding failure. diff --git a/test/performance/generate_report.py b/test/performance/generate_report.py index 14030657..aee53864 100644 --- a/test/performance/generate_report.py +++ b/test/performance/generate_report.py @@ -76,7 +76,7 @@ def _bar_chart_svg(chart_id, title, groups, sizes, width=700, bar_h=28, gap=6): lx = label_col_w for g in groups: svg.append(f'') - svg.append(f'{g["label"]}') + svg.append(f'{g["label"]}') lx += len(g["label"]) * 7 + 30 y = 58 @@ -150,7 +150,7 @@ def _histogram_svg(chart_id, title, series_list, width=700, height=220, n_bins=1 lx = margin_l for s in series_list: svg.append(f'') - svg.append(f'{s["label"]}') + svg.append(f'{s["label"]}') lx += len(s["label"]) * 6 + 24 # Axes @@ -302,14 +302,14 @@ def _build_table(results): durations_str = ", ".join(_fmt(v) for v in d) rows += f""" - {r['test']} - {r['size_mb']} MB - {r['rounds']} + {r["test"]} + {r["size_mb"]} MB + {r["rounds"]} {_fmt(med)} - {_fmt(r['mean_s'])} + {_fmt(r["mean_s"])} {_fmt(p95)} - {_fmt(r['min_s'])} - {_fmt(r['max_s'])} + {_fmt(r["min_s"])} + {_fmt(r["max_s"])} {durations_str} """ @@ -366,9 +366,9 @@ def generate_html(data: dict) -> str:

S3 Encryption Client — Performance Report

Generated: {timestamp}
- Rounds per test: {config['num_rounds']} · - Object sizes: {', '.join(str(s) + ' MB' for s in sizes)} · - Bucket: {config['bucket']} · Region: {config['region']} + Rounds per test: {config["num_rounds"]} · + Object sizes: {", ".join(str(s) + " MB" for s in sizes)} · + Bucket: {config["bucket"]} · Region: {config["region"]}
diff --git a/test/test_stream.py b/test/test_stream.py index dc0f2eb9..692c8b00 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -124,7 +124,6 @@ def spy_finalize(data): class TestDelayedAuthCBCDecryption: - def test_roundtrip(self): plaintext = b"hello world, this is a CBC test!!" ciphertext, key, iv = _encrypt_cbc(plaintext) @@ -242,7 +241,6 @@ def test_empty_ciphertext(self): class TestBufferedDecryptingStream: - def test_full_read(self): plaintext = os.urandom(1024) ct, key, nonce = _encrypt_gcm(plaintext) @@ -379,7 +377,6 @@ def test_idempotent_decrypt(self): class TestDelayedAuthGCMDecryption: - def test_full_read(self): plaintext = os.urandom(1024) ct, key, nonce = _encrypt_gcm(plaintext) @@ -511,7 +508,6 @@ def test_large_data(self): class TestEdgeCasePlaintextLengths: - @pytest.mark.parametrize("length", EDGE_CASE_LENGTHS) def test_buffered_gcm(self, length): plaintext = os.urandom(length) From d6edde6afec713111c47d3275119f44e5307ae53 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Thu, 14 May 2026 12:35:47 -0700 Subject: [PATCH 77/81] chore(test): fix flaky tests, add more platforms (#183) --- .github/workflows/python-integ.yml | 33 ++++++------- .../test_i_delayed_auth_streaming_example.py | 10 ++-- .../test_i_kms_keyring_put_get_example.py | 10 ++-- .../test/test_i_legacy_decrypt_example.py | 1 + pyproject.toml | 4 +- test/integration/test_i_security.py | 46 ++++++++++++------- 6 files changed, 61 insertions(+), 43 deletions(-) diff --git a/.github/workflows/python-integ.yml b/.github/workflows/python-integ.yml index e2d710cc..6a23a7b3 100644 --- a/.github/workflows/python-integ.yml +++ b/.github/workflows/python-integ.yml @@ -4,14 +4,19 @@ on: workflow_call: inputs: python-version: - description: "Python version to use" + description: "Python version to use (ignored when matrix is used)" default: "3.11" required: false type: string jobs: python-integ: - runs-on: macos-14-large + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} permissions: id-token: write contents: read @@ -25,14 +30,16 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: ${{ inputs.python-version || '3.11' }} - + python-version: ${{ matrix.python-version }} + allow-prereleases: true + - name: Cache uv dependencies uses: actions/cache@v5 with: path: ~/.cache/uv - key: ${{ runner.os }}-uv-${{ hashFiles('./test-server/python-v4-server/**/pyproject.toml') }} + key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} restore-keys: | + ${{ runner.os }}-uv-py${{ matrix.python-version }}- ${{ runner.os }}-uv- - name: Install Uv @@ -48,16 +55,10 @@ jobs: aws-region: us-west-2 - name: Run unit tests - run: | - uv run pytest test/ --ignore=test/integration/ --ignore=test/performance/ --verbose \ - --cov=src/s3_encryption --cov-report=term-missing --cov-report=html:coverage-unit \ - --cov-fail-under=89 + run: uv run pytest test/ --ignore=test/integration/ --ignore=test/performance/ --verbose --cov=src/s3_encryption --cov-report=term-missing --cov-report=html:coverage-unit --cov-fail-under=89 - name: Run integration tests - run: | - uv run pytest test/integration/ --verbose \ - --cov=src/s3_encryption --cov-report=term-missing --cov-report=html:coverage-integ \ - --cov-fail-under=83 + run: uv run pytest test/integration/ --verbose --cov=src/s3_encryption --cov-report=term-missing --cov-report=html:coverage-integ --cov-fail-under=83 env: CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} @@ -67,16 +68,16 @@ jobs: - name: Run examples run: make test-examples - - name: Generate coverage HTML report + - name: Upload unit test coverage report if: always() uses: actions/upload-artifact@v7 with: - name: coverage-unit + name: coverage-unit-py${{ matrix.python-version }}-${{ matrix.os }} path: coverage-unit/ - name: Upload integration test coverage report if: always() uses: actions/upload-artifact@v7 with: - name: coverage-integ + name: coverage-integ-py${{ matrix.python-version }}-${{ matrix.os }} path: coverage-integ/ diff --git a/examples/test/test_i_delayed_auth_streaming_example.py b/examples/test/test_i_delayed_auth_streaming_example.py index 501c7be0..d087789c 100644 --- a/examples/test/test_i_delayed_auth_streaming_example.py +++ b/examples/test/test_i_delayed_auth_streaming_example.py @@ -1,6 +1,9 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 """Test suite for the delayed auth streaming decrypt example.""" + +import uuid + import boto3 import pytest @@ -9,11 +12,11 @@ pytestmark = [pytest.mark.examples] BUCKET = "s3ec-python-github-test-bucket" -KEY = "examples/delayed-auth-streaming" KMS_KEY_ID = "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" def test_delayed_auth_streaming_decrypt(): + key = f"examples/delayed-auth-streaming-{uuid.uuid4()}" s3_client = boto3.client("s3", region_name="us-west-2") kms_client = boto3.client("kms", region_name="us-west-2") delayed_auth_streaming_decrypt( @@ -21,7 +24,6 @@ def test_delayed_auth_streaming_decrypt(): kms_client=kms_client, kms_key_id=KMS_KEY_ID, bucket=BUCKET, - key=KEY, + key=key, ) - # Clean up - s3_client.delete_object(Bucket=BUCKET, Key=KEY) + s3_client.delete_object(Bucket=BUCKET, Key=key) diff --git a/examples/test/test_i_kms_keyring_put_get_example.py b/examples/test/test_i_kms_keyring_put_get_example.py index 08759041..bff0e76f 100644 --- a/examples/test/test_i_kms_keyring_put_get_example.py +++ b/examples/test/test_i_kms_keyring_put_get_example.py @@ -1,6 +1,9 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 """Test suite for the KMS Keyring put/get example.""" + +import uuid + import boto3 import pytest @@ -9,11 +12,11 @@ pytestmark = [pytest.mark.examples] BUCKET = "s3ec-python-github-test-bucket" -KEY = "examples/kms-keyring-put-get" KMS_KEY_ID = "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" def test_kms_keyring_put_get(): + key = f"examples/kms-keyring-put-get-{uuid.uuid4()}" s3_client = boto3.client("s3", region_name="us-west-2") kms_client = boto3.client("kms", region_name="us-west-2") kms_keyring_put_get( @@ -21,7 +24,6 @@ def test_kms_keyring_put_get(): kms_client=kms_client, kms_key_id=KMS_KEY_ID, bucket=BUCKET, - key=KEY, + key=key, ) - # Clean up - s3_client.delete_object(Bucket=BUCKET, Key=KEY) + s3_client.delete_object(Bucket=BUCKET, Key=key) diff --git a/examples/test/test_i_legacy_decrypt_example.py b/examples/test/test_i_legacy_decrypt_example.py index 93f67d0c..b072d561 100644 --- a/examples/test/test_i_legacy_decrypt_example.py +++ b/examples/test/test_i_legacy_decrypt_example.py @@ -1,6 +1,7 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 """Test suite for the legacy decrypt example.""" + import boto3 import pytest diff --git a/pyproject.toml b/pyproject.toml index 7fb8a58e..25318942 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ authors = [ ] license = {text = "Apache-2.0"} readme = "README.md" -requires-python = ">=3.11" +requires-python = ">=3.10" dependencies = [ "boto3>=1.43.6,<2", "cryptography>=48.0.0,<49", @@ -33,7 +33,7 @@ packages = ["src/s3_encryption"] [tool.ruff] line-length = 100 -target-version = "py311" +target-version = "py310" exclude = [".git", "__pycache__", "build", "dist"] [tool.ruff.lint] diff --git a/test/integration/test_i_security.py b/test/integration/test_i_security.py index 3d871e1f..4d73a3ae 100644 --- a/test/integration/test_i_security.py +++ b/test/integration/test_i_security.py @@ -462,11 +462,18 @@ def test_wrong_key_and_tampered_ciphertext_produce_same_error(self): iv = os.urandom(16) ciphertext = self._encrypt_cbc(key, iv, b"test data for padding oracle check") - # Wrong key: decryption produces garbage, unpadding fails - wrong_key = os.urandom(32) - decryptor1 = self._make_cbc_decryptor(wrong_key, iv, len(ciphertext)) - with pytest.raises(S3EncryptionClientSecurityError) as exc1: - decryptor1.finalize(ciphertext) + # Wrong key: decryption produces garbage, unpadding fails. + # ~1/256 chance random garbage has valid PKCS7 padding, so retry. + exc1 = None + for _ in range(10): + wrong_key = os.urandom(32) + decryptor1 = self._make_cbc_decryptor(wrong_key, iv, len(ciphertext)) + try: + decryptor1.finalize(ciphertext) + except S3EncryptionClientSecurityError as e: + exc1 = e + break + assert exc1 is not None, "Wrong key did not produce padding error after 10 attempts" # Tampered ciphertext: last byte flipped, unpadding fails tampered = ciphertext[:-1] + bytes([ciphertext[-1] ^ 0x01]) @@ -475,13 +482,13 @@ def test_wrong_key_and_tampered_ciphertext_produce_same_error(self): decryptor2.finalize(tampered) # Both MUST produce the same error message - assert str(exc1.value) == str(exc2.value), ( - f"Error messages differ: wrong_key={str(exc1.value)!r}, tampered={str(exc2.value)!r}" + assert str(exc1) == str(exc2.value), ( + f"Error messages differ: wrong_key={str(exc1)!r}, tampered={str(exc2.value)!r}" ) # Neither message should contain details about the underlying failure - assert "padding" not in str(exc1.value).lower(), ( - f"Error message leaks padding information: {str(exc1.value)!r}" + assert "padding" not in str(exc1).lower(), ( + f"Error message leaks padding information: {str(exc1)!r}" ) def test_truncated_ciphertext_produces_same_error(self): @@ -495,11 +502,17 @@ def test_truncated_ciphertext_produces_same_error(self): iv = os.urandom(16) ciphertext = self._encrypt_cbc(key, iv, b"test data for truncation check") - # Padding failure (wrong key) - wrong_key = os.urandom(32) - decryptor1 = self._make_cbc_decryptor(wrong_key, iv, len(ciphertext)) - with pytest.raises(S3EncryptionClientSecurityError) as exc1: - decryptor1.finalize(ciphertext) + # Padding failure (wrong key) — retry for same reason as above + exc1 = None + for _ in range(10): + wrong_key = os.urandom(32) + decryptor1 = self._make_cbc_decryptor(wrong_key, iv, len(ciphertext)) + try: + decryptor1.finalize(ciphertext) + except S3EncryptionClientSecurityError as e: + exc1 = e + break + assert exc1 is not None, "Wrong key did not produce padding error after 10 attempts" # Truncated ciphertext (not block-aligned) truncated = ciphertext[:-3] @@ -508,9 +521,8 @@ def test_truncated_ciphertext_produces_same_error(self): decryptor2.finalize(truncated) # Both MUST produce the same error message - assert str(exc1.value) == str(exc2.value), ( - f"Error messages differ: padding_fail={str(exc1.value)!r}, " - f"truncated={str(exc2.value)!r}" + assert str(exc1) == str(exc2.value), ( + f"Error messages differ: padding_fail={str(exc1)!r}, truncated={str(exc2.value)!r}" ) From d158772802c78b2d160b2ed5f6715ac5ebb58f77 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Fri, 15 May 2026 12:24:04 -0700 Subject: [PATCH 78/81] chore: update README etc (#185) --- NOTICE | 1 + README.md | 81 +++++++++++++++++++++++++---------------- SUPPORT_POLICY.rst | 4 +- test/test_exceptions.py | 2 + 4 files changed, 54 insertions(+), 34 deletions(-) diff --git a/NOTICE b/NOTICE index 616fc588..3d299504 100644 --- a/NOTICE +++ b/NOTICE @@ -1 +1,2 @@ +Amazon S3 Encryption Client for Python Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/README.md b/README.md index edcfd4ff..70866688 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,38 @@ -# Amazon S3 Encryption Client Python +# Amazon S3 Encryption Client for Python -This library provides an S3 client that supports client-side encryption. +This library provides an S3 client that supports client-side encryption. For more information and detailed instructions for how to use this library, refer to the [Amazon S3 Encryption Client Developer Guide](https://docs.aws.amazon.com/amazon-s3-encryption-client/latest/developerguide/python.html). + +## Getting Started + +Requires Python 3.10 or greater. An AWS account is required; standard S3 and KMS charges apply. + +The S3 Encryption Client wraps a standard boto3 S3 client and uses a KMS keyring to manage data key encryption. Objects are encrypted before upload and decrypted after download transparently. By default, the client uses AES-GCM with key commitment for content encryption. + +```python +import boto3 +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.materials.kms_keyring import KmsKeyring + +kms_client = boto3.client("kms", region_name="us-west-2") +keyring = KmsKeyring(kms_client, "arn:aws:kms:us-west-2:123456789012:alias/my-key") + +s3_client = boto3.client("s3") +config = S3EncryptionClientConfig(keyring=keyring) +s3ec = S3EncryptionClient(s3_client, config) + +# Encrypt and upload +s3ec.put_object(Bucket="my-bucket", Key="my-object", Body=b"secret data") + +# Download and decrypt +response = s3ec.get_object(Bucket="my-bucket", Key="my-object") +plaintext = response["Body"].read() +``` ## Development ### Prerequisites -- Python 3.11 or higher +- Python 3.10 or higher - [uv](https://github.com/astral-sh/uv) for package and project management ### Setup @@ -19,7 +45,7 @@ make install ### Testing -Run all tests: +Run all tests (unit + integration + examples): ```bash make test @@ -39,47 +65,38 @@ make test-integration ### Code Quality -This project uses [Black](https://black.readthedocs.io/) for code formatting, [isort](https://pycqa.github.io/isort/) for import sorting, and [Flake8](https://flake8.pycqa.org/) for linting. +This project uses [Ruff](https://docs.astral.sh/ruff/) for linting and formatting. -Check code quality: +Check formatting: ```bash -make lint +make format-check ``` -Format code with Black and isort: +Run linter: ```bash -make format +make lint ``` -Clean up cache files: +Format code and auto-fix lint issues: ```bash -make clean +make format ``` -#### Linting Standards - -The project is configured with Black, isort, and Flake8 to enforce consistent code style and quality. Currently, Flake8 is set to report issues but not fail the build, allowing for gradual adoption of linting standards. +### Integration Test Resources -Common Flake8 issues in the codebase include: +Integration tests require AWS credentials and the following resources. The tests use environment variables to override CI defaults: -- **Missing docstrings** (D100-D104): Add docstrings to modules, classes, and functions -- **Docstring formatting** (D200, D212, D415): Follow Google docstring style -- **Line length** (E501): Keep lines under 100 characters -- **Unused imports** (F401): Remove unused imports -- **Unused variables** (F841): Remove or use assigned variables -- **Code complexity** (C901): Refactor complex functions - -When contributing to this project, please try to fix linting issues in the files you modify. - -### Pull Request Command -While this project is in development, -it is useful to use `gh pr` to create the pull-requests, -so they can be associated with the GitHub project. - -```sh -gh pr create -B staging -p "S3EC-Python" -f -``` +| Variable | Description | Default | +|----------|-------------|---------| +| `CI_S3_BUCKET` | S3 bucket for read/write tests | `s3ec-python-github-test-bucket` | +| `CI_AWS_REGION` | Primary AWS region | `us-west-2` | +| `CI_KMS_KEY_ALIAS` | KMS key ARN or alias for encryption | `arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key` | +| `CI_MRK_KEY_ID_PRIMARY` | Multi-region key ARN (primary region) | `arn:aws:kms:us-west-2:370957321024:key/mrk-cea4cf67c6a046ba829f61f69db5c191` | +| `CI_MRK_KEY_ID_REPLICA` | Multi-region key ARN (replica region) | `arn:aws:kms:us-east-1:370957321024:key/mrk-cea4cf67c6a046ba829f61f69db5c191` | +| `CI_S3_STATIC_TEST_BUCKET` | Bucket with pre-existing test objects for instruction file tests | `s3ec-static-test-objects` | +| `CI_KMS_KEY_STATIC_TESTS` | KMS key used for static test objects | `arn:aws:kms:us-west-2:370957321024:key/a3889cd9-99eb-4138-a93a-aea9d52ec2ef` | +To run integration tests locally, configure AWS credentials with access to these resources (or your own equivalents) and set the environment variables accordingly. diff --git a/SUPPORT_POLICY.rst b/SUPPORT_POLICY.rst index 3eafd39b..5920fb8b 100644 --- a/SUPPORT_POLICY.rst +++ b/SUPPORT_POLICY.rst @@ -21,9 +21,9 @@ This table describes the current support status of each major version of the Ama - Current status - Next status - Next status date - * - 3.x - - Pre-Release + * - 4.x - Generally Available - + - .. _AWS SDKs and Tools Maintenance Policy: https://docs.aws.amazon.com/sdkref/latest/guide/maint-policy.html#version-life-cycle diff --git a/test/test_exceptions.py b/test/test_exceptions.py index 4fe46bcc..84fec0d0 100644 --- a/test/test_exceptions.py +++ b/test/test_exceptions.py @@ -1,3 +1,5 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 import pytest from botocore.exceptions import BotoCoreError From ddf8b677042617f6a4ad238b4be6a660756eaa42 Mon Sep 17 00:00:00 2001 From: Tony Knapp <5892063+texastony@users.noreply.github.com> Date: Mon, 18 May 2026 15:40:35 -0500 Subject: [PATCH 79/81] feat: add low-level Multipart Upload and upload_file(obj) (#172) Co-authored-by: Kess Plasmeier --- src/s3_encryption/__init__.py | 303 +++++- src/s3_encryption/pipelines.py | 137 +++ .../test_i_s3_encryption_multipart.py | 968 ++++++++++++++++++ .../test_i_s3_encryption_multithreaded.py | 106 ++ .../test_i_s3_encryption_transfer_manager.py | 396 +++++++ test/test_multipart.py | 780 ++++++++++++++ test/test_stream.py | 22 +- 7 files changed, 2703 insertions(+), 9 deletions(-) create mode 100644 test/integration/test_i_s3_encryption_multipart.py create mode 100644 test/integration/test_i_s3_encryption_transfer_manager.py create mode 100644 test/test_multipart.py diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 3f188510..e06ca9e1 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -3,6 +3,7 @@ """Top-level S3 Encryption Client v4 for Python package.""" import io +import os import threading from attrs import define, field @@ -19,10 +20,19 @@ ) from .materials.keyring import AbstractKeyring from .materials.materials import AlgorithmSuite, CommitmentPolicy -from .pipelines import GetEncryptedObjectPipeline, PutEncryptedObjectPipeline +from .pipelines import ( + GetEncryptedObjectPipeline, + MultipartUploadPipeline, + PutEncryptedObjectPipeline, +) S3_METADATA_PREFIX = "x-amz-meta-" +# Default multipart threshold and chunk size (same as boto3 defaults) +_DEFAULT_MULTIPART_THRESHOLD = 8 * 1024 * 1024 # 8 MB +_DEFAULT_MULTIPART_CHUNKSIZE = 8 * 1024 * 1024 # 8 MB +_MIN_MULTIPART_PART_SIZE = 5 * 1024 * 1024 # 5 MB — S3 minimum for non-final parts + # Thread-local context attribute names _CTX_ENCRYPTION_CONTEXT = "encryption_context" _CTX_BUCKET = "bucket" @@ -341,6 +351,10 @@ class S3EncryptionClient: wrapped_s3_client = field() config: S3EncryptionClientConfig = field() _plugin: S3EncryptionClientPlugin = field(init=False) + # Each upload gets its own pipeline with independent cipher state, keyed by UploadId. + # Access is protected by a lock for thread safety across all Python runtimes. + _multipart_uploads: dict = field(init=False, factory=dict) + _multipart_lock: threading.Lock = field(init=False, factory=threading.Lock) def __attrs_post_init__(self): """Install the encryption plugin on the wrapped client using boto3 events.""" @@ -563,3 +577,290 @@ def get_object(self, **kwargs): for attr in _GET_OBJECT_CLEANUP_ATTRS: if hasattr(self._plugin._context, attr): delattr(self._plugin._context, attr) + + ##= specification/s3-encryption/client.md#optional-api-operations + ##= type=implementation + ##% CreateMultipartUpload MAY be implemented by the S3EC. + def create_multipart_upload(self, **kwargs): + """Initiate an encrypted multipart upload. + + Obtains encryption materials, initializes the cipher, and calls + the underlying S3 CreateMultipartUpload. Encryption metadata is + set on the object at this point. + + Args: + **kwargs: Arguments for S3 create_multipart_upload. + May include EncryptionContext. + + Returns: + The response from S3 create_multipart_upload. + """ + encryption_context = kwargs.pop("EncryptionContext", None) + _validate_encryption_context(encryption_context) + + pipeline = MultipartUploadPipeline( + cmm=self.config.cmm, + encryption_algorithm=self.config.encryption_algorithm, + encryption_context=encryption_context or {}, + ) + + # Merge encryption metadata into user-provided Metadata + user_metadata = dict(kwargs.get("Metadata", {})) + user_metadata.update(pipeline.metadata) + kwargs["Metadata"] = user_metadata + + ##= specification/s3-encryption/client.md#optional-api-operations + ##= type=implementation + ##% If implemented, CreateMultipartUpload MUST initiate a multipart upload. + try: + response = self.wrapped_s3_client.create_multipart_upload(**kwargs) + except Exception as e: + raise S3EncryptionClientError(f"Failed to create multipart upload: {e}") from e + + upload_id = response["UploadId"] + with self._multipart_lock: + self._multipart_uploads[upload_id] = pipeline + return response + + ##= specification/s3-encryption/client.md#optional-api-operations + ##= type=implementation + ##% UploadPart MAY be implemented by the S3EC. + ##= specification/s3-encryption/client.md#optional-api-operations + ##= type=implementation + ##% UploadPart MUST encrypt each part. + ##= specification/s3-encryption/client.md#optional-api-operations + ##= type=implementation + ##% Each part MUST be encrypted in sequence. + ##= specification/s3-encryption/client.md#optional-api-operations + ##= type=implementation + ##% Each part MUST be encrypted using the same cipher instance for each part. + def upload_part(self, **kwargs): + """Encrypt and upload a single part of a multipart upload. + + Parts must be uploaded in sequential order (1, 2, 3, ...). + The caller MUST set ``IsLastPart=True`` on the final part so the + GCM authentication tag is appended to the ciphertext. + + Args: + **kwargs: Arguments for S3 upload_part. Must include UploadId, + PartNumber, and Body. Set IsLastPart=True on the + final part. + + Returns: + The response from S3 upload_part (includes ETag). + """ + upload_id = kwargs.get("UploadId") + with self._multipart_lock: + pipeline = self._multipart_uploads.get(upload_id) + if pipeline is None: + raise S3EncryptionClientError( + f"No multipart upload found for UploadId: {upload_id}. " + "Call create_multipart_upload first." + ) + + part_number = kwargs["PartNumber"] + is_last = kwargs.pop("IsLastPart", False) + body = kwargs.get("Body", b"") + if isinstance(body, str): + body = body.encode("utf-8") + elif hasattr(body, "read"): + body = body.read() + + try: + ciphertext = pipeline.encrypt_part(part_number, body, is_last=is_last) + except S3EncryptionClientError: + raise + except Exception as e: + raise S3EncryptionClientError(f"Failed to encrypt part {part_number}: {e}") from e + + kwargs["Body"] = ciphertext + return self.wrapped_s3_client.upload_part(**kwargs) + + ##= specification/s3-encryption/client.md#optional-api-operations + ##= type=implementation + ##% CompleteMultipartUpload MAY be implemented by the S3EC. + ##% CompleteMultipartUpload MUST complete the multipart upload. + def complete_multipart_upload(self, **kwargs): + """Complete the multipart upload. + + The final part must have been uploaded with ``IsLastPart=True`` + before calling this method. + + Args: + **kwargs: Arguments for S3 complete_multipart_upload. + MultipartUpload.Parts must include PartNumber and ETag + for each part. + + Returns: + The response from S3 complete_multipart_upload. + """ + upload_id = kwargs.get("UploadId") + with self._multipart_lock: + pipeline = self._multipart_uploads.get(upload_id) + if pipeline is None: + raise S3EncryptionClientError(f"No multipart upload found for UploadId: {upload_id}.") + + if not pipeline.has_final_part_been_seen: + raise S3EncryptionClientError( + "Cannot complete multipart upload: the final part has not been uploaded. " + "Set IsLastPart=True on the last upload_part call." + ) + + try: + response = self.wrapped_s3_client.complete_multipart_upload(**kwargs) + except S3EncryptionClientError: + raise + except Exception as e: + raise S3EncryptionClientError(f"Failed to complete multipart upload: {e}") from e + else: + with self._multipart_lock: + self._multipart_uploads.pop(upload_id, None) + return response + + ##= specification/s3-encryption/client.md#optional-api-operations + ##= type=implementation + ##% AbortMultipartUpload MAY be implemented by the S3EC. + ##% AbortMultipartUpload MUST abort the multipart upload. + def abort_multipart_upload(self, **kwargs): + """Abort a multipart upload and clean up cipher state. + + Args: + **kwargs: Arguments for S3 abort_multipart_upload. + + Returns: + The response from S3 abort_multipart_upload. + """ + upload_id = kwargs.get("UploadId") + with self._multipart_lock: + self._multipart_uploads.pop(upload_id, None) + return self.wrapped_s3_client.abort_multipart_upload(**kwargs) + + def upload_file( + self, filename, bucket, key, multipart_threshold=None, multipart_chunksize=None, **kwargs + ): + """Encrypt and upload a file to S3. + + If the file is smaller than the threshold, uses put_object. + Otherwise, performs an encrypted multipart upload. + + Args: + filename: Path to the file to upload. + bucket: Target S3 bucket. + key: Target S3 object key. + multipart_threshold: File size threshold for multipart (default 8MB). + multipart_chunksize: Size of each part (default 8MB). + **kwargs: Additional arguments (e.g. EncryptionContext, Metadata). + """ + threshold = ( + _DEFAULT_MULTIPART_THRESHOLD if multipart_threshold is None else multipart_threshold + ) + chunksize = ( + _DEFAULT_MULTIPART_CHUNKSIZE if multipart_chunksize is None else multipart_chunksize + ) + if threshold <= 0: + raise S3EncryptionClientError("multipart_threshold must be a positive integer.") + if chunksize <= 0: + raise S3EncryptionClientError("multipart_chunksize must be a positive integer.") + if chunksize < _MIN_MULTIPART_PART_SIZE: + raise S3EncryptionClientError( + f"multipart_chunksize must be at least {_MIN_MULTIPART_PART_SIZE} bytes (5 MB). " + f"S3 requires all non-final parts to be at least 5 MB." + ) + file_size = os.path.getsize(filename) + + if file_size < threshold: + with open(filename, "rb") as f: + kwargs["Bucket"] = bucket + kwargs["Key"] = key + kwargs["Body"] = f.read() + return self.put_object(**kwargs) + + return self._multipart_upload_from_readable( + open(filename, "rb"), bucket, key, chunksize, owns_readable=True, **kwargs + ) + + def upload_fileobj(self, fileobj, bucket, key, multipart_chunksize=None, **kwargs): + """Encrypt and upload a file-like object to S3 via multipart upload. + + The caller retains ownership of fileobj — it will not be closed + by this method. + + Args: + fileobj: A file-like object with a read() method. + bucket: Target S3 bucket. + key: Target S3 object key. + multipart_chunksize: Size of each part (default 8MB). + **kwargs: Additional arguments (e.g. EncryptionContext, Metadata). + """ + chunksize = ( + _DEFAULT_MULTIPART_CHUNKSIZE if multipart_chunksize is None else multipart_chunksize + ) + if chunksize <= 0: + raise S3EncryptionClientError("multipart_chunksize must be a positive integer.") + if chunksize < _MIN_MULTIPART_PART_SIZE: + raise S3EncryptionClientError( + f"multipart_chunksize must be at least {_MIN_MULTIPART_PART_SIZE} bytes (5 MB). " + f"S3 requires all non-final parts to be at least 5 MB." + ) + return self._multipart_upload_from_readable( + fileobj, bucket, key, chunksize, owns_readable=False, **kwargs + ) + + def _multipart_upload_from_readable( + self, readable, bucket, key, chunksize, *, owns_readable=False, **kwargs + ): + """Perform an encrypted multipart upload from a readable source. + + Args: + readable: File-like object to read from. + bucket: Target S3 bucket. + key: Target S3 object key. + chunksize: Size of each part in bytes. + owns_readable: If True, close readable when done. If False, + the caller is responsible for closing it. + **kwargs: Additional S3 parameters forwarded to create_multipart_upload. + """ + # EncryptionContext is consumed by our pipeline, not S3 + create_kwargs = {"Bucket": bucket, "Key": key} + if "EncryptionContext" in kwargs: + create_kwargs["EncryptionContext"] = kwargs.pop("EncryptionContext") + if "Metadata" in kwargs: + create_kwargs["Metadata"] = kwargs.pop("Metadata") + # Forward remaining kwargs (ACL, ContentType, Tagging, etc.) to create_multipart_upload + create_kwargs.update(kwargs) + + create_resp = self.create_multipart_upload(**create_kwargs) + upload_id = create_resp["UploadId"] + + try: + parts = [] + part_number = 0 + # Read ahead so we can detect the last chunk + current_chunk = readable.read(chunksize) + while current_chunk: + next_chunk = readable.read(chunksize) + part_number += 1 + is_last = not next_chunk + resp = self.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=part_number, + Body=current_chunk, + IsLastPart=is_last, + ) + parts.append({"PartNumber": part_number, "ETag": resp["ETag"]}) + current_chunk = next_chunk + + return self.complete_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={"Parts": parts}, + ) + except Exception: + self.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + raise + finally: + if owns_readable and hasattr(readable, "close"): + readable.close() diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 23a169ae..ca200a7f 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -9,6 +9,7 @@ import base64 import json import os +import threading from attrs import define, field from botocore.response import StreamingBody @@ -171,6 +172,142 @@ def _encrypt_kc_gcm(self, plaintext, enc_mats, edk_bytes): return encrypted_data, metadata.to_dict() +##= specification/s3-encryption/client.md#optional-api-operations +##= type=implementation +##% UploadPart MUST encrypt each part. +##= specification/s3-encryption/client.md#optional-api-operations +##= type=implementation +##% Each part MUST be encrypted in sequence. +##= specification/s3-encryption/client.md#optional-api-operations +##= type=implementation +##% Each part MUST be encrypted using the same cipher instance for each part. +@define +class MultipartUploadPipeline: + """Pipeline for encrypting multipart uploads. + + Manages a single AES-GCM cipher instance shared across all parts. + Parts MUST be uploaded in sequence (1, 2, 3, ...). + """ + + cmm: AbstractCryptoMaterialsManager = field() + encryption_algorithm: AlgorithmSuite = field() + encryption_context: dict = field(factory=dict) + _encryptor: object = field(init=False, default=None) + _metadata: dict = field(init=False, factory=dict) + _next_part: int = field(init=False, default=1) + _has_final_part_been_seen: bool = field(init=False, default=False) + _lock: threading.Lock = field(init=False, factory=threading.Lock) + # Cached ciphertext for the most recently encrypted part, enabling retries + # if the S3 upload_part call fails after encryption has already advanced. + _last_encrypted_part: int = field(init=False, default=0) + _last_encrypted_ciphertext: bytes | None = field(init=False, default=None) + + def __attrs_post_init__(self): + """Obtain encryption materials and initialize the streaming cipher.""" + enc_mats_request = EncryptionMaterials( + encryption_algorithm=self.encryption_algorithm, + encryption_context=self.encryption_context.copy(), + ) + enc_mats = self.cmm.get_encryption_materials(enc_mats_request) + if enc_mats.plaintext_data_key is None: + raise S3EncryptionClientError("No plaintext data key found!") + if enc_mats.encrypted_data_key is None: + raise S3EncryptionClientError("No encrypted data key found!") + + edk_bytes = enc_mats.encrypted_data_key.encrypted_data_key + + if self.encryption_algorithm == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + self._init_kc_gcm(enc_mats, edk_bytes) + else: + self._init_gcm(enc_mats, edk_bytes) + + def _init_gcm(self, enc_mats, edk_bytes): + iv = os.urandom(enc_mats.encryption_algorithm.cipher_iv_length_bytes) + cipher = Cipher(algorithms.AES(enc_mats.plaintext_data_key), modes.GCM(iv)) + self._encryptor = cipher.encryptor() + self._metadata = ObjectMetadata( + encrypted_data_key_v2=base64.b64encode(edk_bytes).decode("utf-8"), + encrypted_data_key_algorithm="kms+context", + content_iv=base64.b64encode(iv).decode("utf-8"), + content_cipher="AES/GCM/NoPadding", + encrypted_data_key_context=enc_mats.encryption_context, + ).to_dict() + + def _init_kc_gcm(self, enc_mats, edk_bytes): + algorithm_suite = enc_mats.encryption_algorithm + message_id = os.urandom(algorithm_suite.commitment_nonce_length_bytes) + derived_encryption_key, commit_key = derive_keys( + enc_mats.plaintext_data_key, message_id, algorithm_suite + ) + cipher = Cipher( + algorithms.AES(derived_encryption_key), modes.GCM(algorithm_suite.kc_gcm_iv) + ) + self._encryptor = cipher.encryptor() + self._encryptor.authenticate_additional_data(algorithm_suite.suite_id_bytes) + self._metadata = ObjectMetadata( + content_cipher_v3=str(algorithm_suite.suite_id), + encrypted_data_key_algorithm_v3="12", + encrypted_data_key_v3=base64.b64encode(edk_bytes).decode("utf-8"), + message_id_v3=base64.b64encode(message_id).decode("utf-8"), + key_commitment_v3=base64.b64encode(commit_key).decode("utf-8"), + encryption_context_v3=( + enc_mats.encryption_context if enc_mats.encryption_context else None + ), + ).to_dict() + + @property + def metadata(self): + """Return the encryption metadata dict for the multipart upload.""" + return self._metadata + + @property + def has_final_part_been_seen(self): + """Return whether the final part has been encrypted.""" + return self._has_final_part_been_seen + + def encrypt_part(self, part_number, data, is_last=False): + """Encrypt a single part. Parts must be sequential starting from 1. + + If called with the same part_number as the most recently encrypted part, + returns the cached ciphertext (enabling retries after upload failures). + + Args: + part_number: The 1-based part number. + data: The plaintext bytes for this part. + is_last: If True, finalizes the cipher and appends the GCM auth tag. + + Returns: + The encrypted ciphertext bytes for this part. + """ + with self._lock: + # Allow retry of the last encrypted part + if part_number == self._last_encrypted_part: + return self._last_encrypted_ciphertext + + if self._has_final_part_been_seen: + raise S3EncryptionClientError("Cannot encrypt more parts after the final part.") + if part_number != self._next_part: + raise S3EncryptionClientError( + f"Parts must be uploaded in sequence. Expected part {self._next_part}, " + f"got {part_number}." + ) + if isinstance(data, str): + data = data.encode("utf-8") + self._next_part += 1 + + ciphertext = self._encryptor.update(data) + + if is_last: + self._encryptor.finalize() + ciphertext += self._encryptor.tag + self._has_final_part_been_seen = True + + self._last_encrypted_part = part_number + self._last_encrypted_ciphertext = ciphertext + + return ciphertext + + @define class GetEncryptedObjectPipeline: """Pipeline for decrypting objects after they are retrieved from S3. diff --git a/test/integration/test_i_s3_encryption_multipart.py b/test/integration/test_i_s3_encryption_multipart.py new file mode 100644 index 00000000..1aefef31 --- /dev/null +++ b/test/integration/test_i_s3_encryption_multipart.py @@ -0,0 +1,968 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Integration tests for encrypted multipart upload. + +These tests verify that the S3 Encryption Client correctly encrypts +objects via multipart upload and that they can be decrypted via get_object. +Tests cover the low-level multipart API (create/upload_part/complete/abort) +and the high-level upload_file / upload_fileobj convenience methods. +""" + +import os +import threading +from datetime import datetime + +import boto3 +import pytest + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy + +bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") +region = os.environ.get("CI_AWS_REGION", "us-west-2") +kms_key_id = os.environ.get( + "CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" +) + +ALGORITHM_CONFIGS = [ + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + id="AES_GCM", + ), + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + id="KC_GCM", + ), +] + +# Minimum part size for S3 multipart upload is 5 MB (except last part). +FIVE_MB = 5 * 1024 * 1024 + + +def _make_client(algorithm_suite, commitment_policy, **extra_config): + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig( + keyring, + encryption_algorithm=algorithm_suite, + commitment_policy=commitment_policy, + **extra_config, + ) + return S3EncryptionClient(wrapped_client, config) + + +def _unique_key(prefix): + return prefix + datetime.now().strftime("%Y-%m-%d-%H:%M:%S-%f") + + +# --------------------------------------------------------------------------- +# Low-level multipart API: create → upload_part → complete +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_multipart_two_parts_roundtrip(algorithm_suite, commitment_policy): + """Encrypt two 5 MB parts via multipart upload, then decrypt with get_object.""" + key = _unique_key("mpu-2part-") + part1_data = os.urandom(FIVE_MB) + part2_data = os.urandom(1024) # last part can be smaller + expected = part1_data + part2_data + + s3ec = _make_client(algorithm_suite, commitment_policy) + + ##= specification/s3-encryption/client.md#optional-api-operations + ##= type=test + ##% CreateMultipartUpload MAY be implemented by the S3EC. + ##% If implemented, CreateMultipartUpload MUST initiate a multipart upload. + create_resp = s3ec.create_multipart_upload(Bucket=bucket, Key=key) + upload_id = create_resp["UploadId"] + + try: + ##= specification/s3-encryption/client.md#optional-api-operations + ##= type=test + ##% UploadPart MAY be implemented by the S3EC. + resp1 = s3ec.upload_part( + Bucket=bucket, Key=key, UploadId=upload_id, PartNumber=1, Body=part1_data + ) + ##= specification/s3-encryption/client.md#optional-api-operations + ##= type=test + ##% Each part MUST be encrypted in sequence. + resp2 = s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=2, + Body=part2_data, + IsLastPart=True, + ) + + ##= specification/s3-encryption/client.md#optional-api-operations + ##= type=test + ##% CompleteMultipartUpload MUST complete the multipart upload. + s3ec.complete_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={ + "Parts": [ + {"PartNumber": 1, "ETag": resp1["ETag"]}, + {"PartNumber": 2, "ETag": resp2["ETag"]}, + ] + }, + ) + except Exception: + s3ec.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + raise + + ##= specification/s3-encryption/client.md#optional-api-operations + ##= type=test + ##% Each part MUST be encrypted using the same cipher instance for each part. + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == expected + + ##= specification/s3-encryption/client.md#optional-api-operations + ##= type=test + ##% UploadPart MUST encrypt each part. + plain_s3 = boto3.client("s3") + raw_response = plain_s3.get_object(Bucket=bucket, Key=key) + raw_content = raw_response["Body"].read() + assert raw_content != expected + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_multipart_single_part(algorithm_suite, commitment_policy): + """A multipart upload with a single part should still round-trip correctly.""" + key = _unique_key("mpu-1part-") + data = os.urandom(FIVE_MB) + + s3ec = _make_client(algorithm_suite, commitment_policy) + + create_resp = s3ec.create_multipart_upload(Bucket=bucket, Key=key) + upload_id = create_resp["UploadId"] + + try: + resp = s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=1, + Body=data, + IsLastPart=True, + ) + + s3ec.complete_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={"Parts": [{"PartNumber": 1, "ETag": resp["ETag"]}]}, + ) + except Exception: + s3ec.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + raise + + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == data + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_multipart_three_parts(algorithm_suite, commitment_policy): + """Three-part multipart upload: 5MB + 5MB + small last part.""" + key = _unique_key("mpu-3part-") + parts_data = [os.urandom(FIVE_MB), os.urandom(FIVE_MB), os.urandom(2048)] + expected = b"".join(parts_data) + + s3ec = _make_client(algorithm_suite, commitment_policy) + + create_resp = s3ec.create_multipart_upload(Bucket=bucket, Key=key) + upload_id = create_resp["UploadId"] + + try: + parts = [] + for i, part_data in enumerate(parts_data, start=1): + is_last = i == len(parts_data) + resp = s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=i, + Body=part_data, + IsLastPart=is_last, + ) + parts.append({"PartNumber": i, "ETag": resp["ETag"]}) + + s3ec.complete_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={"Parts": parts}, + ) + except Exception: + s3ec.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + raise + + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == expected + + +# --------------------------------------------------------------------------- +# Abort +# --------------------------------------------------------------------------- + + +##= specification/s3-encryption/client.md#optional-api-operations +##= type=test +##% AbortMultipartUpload MAY be implemented by the S3EC. +##% AbortMultipartUpload MUST abort the multipart upload. +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_abort_multipart_upload(algorithm_suite, commitment_policy): + """Aborting a multipart upload should clean up without leaving an object.""" + key = _unique_key("mpu-abort-") + + s3ec = _make_client(algorithm_suite, commitment_policy) + + create_resp = s3ec.create_multipart_upload(Bucket=bucket, Key=key) + upload_id = create_resp["UploadId"] + + # Upload one part then abort + s3ec.upload_part( + Bucket=bucket, Key=key, UploadId=upload_id, PartNumber=1, Body=os.urandom(FIVE_MB) + ) + s3ec.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + + # Object should not exist + plain_s3 = boto3.client("s3") + with pytest.raises(plain_s3.exceptions.NoSuchKey): + plain_s3.get_object(Bucket=bucket, Key=key) + + +# --------------------------------------------------------------------------- +# Encryption context with multipart upload +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_multipart_with_encryption_context(algorithm_suite, commitment_policy): + """Multipart upload with encryption context should be usable on decrypt.""" + key = _unique_key("mpu-ec-") + data = os.urandom(FIVE_MB + 1024) + encryption_context = {"project": "s3ec-python", "test": "multipart"} + + s3ec = _make_client(algorithm_suite, commitment_policy) + + create_resp = s3ec.create_multipart_upload( + Bucket=bucket, Key=key, EncryptionContext=encryption_context + ) + upload_id = create_resp["UploadId"] + + try: + resp1 = s3ec.upload_part( + Bucket=bucket, Key=key, UploadId=upload_id, PartNumber=1, Body=data[:FIVE_MB] + ) + resp2 = s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=2, + Body=data[FIVE_MB:], + IsLastPart=True, + ) + + s3ec.complete_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={ + "Parts": [ + {"PartNumber": 1, "ETag": resp1["ETag"]}, + {"PartNumber": 2, "ETag": resp2["ETag"]}, + ] + }, + ) + except Exception: + s3ec.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + raise + + # Decrypt with matching encryption context + response = s3ec.get_object(Bucket=bucket, Key=key, EncryptionContext=encryption_context) + assert response["Body"].read() == data + + # Decrypt with wrong encryption context should fail + with pytest.raises(S3EncryptionClientError): + s3ec.get_object(Bucket=bucket, Key=key, EncryptionContext={"wrong": "context"}) + + +# --------------------------------------------------------------------------- +# Streaming decryption of multipart-uploaded objects +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_multipart_decrypt_with_delayed_auth(algorithm_suite, commitment_policy): + """Objects uploaded via multipart should be decryptable in delayed-auth mode.""" + key = _unique_key("mpu-delayed-auth-") + data = os.urandom(FIVE_MB + 2048) + + # Encrypt with default (buffered) client + writer = _make_client(algorithm_suite, commitment_policy) + create_resp = writer.create_multipart_upload(Bucket=bucket, Key=key) + upload_id = create_resp["UploadId"] + + try: + resp1 = writer.upload_part( + Bucket=bucket, Key=key, UploadId=upload_id, PartNumber=1, Body=data[:FIVE_MB] + ) + resp2 = writer.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=2, + Body=data[FIVE_MB:], + IsLastPart=True, + ) + + writer.complete_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={ + "Parts": [ + {"PartNumber": 1, "ETag": resp1["ETag"]}, + {"PartNumber": 2, "ETag": resp2["ETag"]}, + ] + }, + ) + except Exception: + writer.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + raise + + # Decrypt with delayed-auth streaming + reader = _make_client(algorithm_suite, commitment_policy, enable_delayed_authentication=True) + response = reader.get_object(Bucket=bucket, Key=key) + + result = b"" + while chunk := response["Body"].read(65536): + result += chunk + assert result == data + + +# --------------------------------------------------------------------------- +# Metadata verification +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_multipart_metadata_present(algorithm_suite, commitment_policy): + """Multipart-uploaded objects should have encryption metadata set.""" + key = _unique_key("mpu-metadata-") + data = os.urandom(FIVE_MB + 512) + + s3ec = _make_client(algorithm_suite, commitment_policy) + + create_resp = s3ec.create_multipart_upload(Bucket=bucket, Key=key) + upload_id = create_resp["UploadId"] + + try: + resp1 = s3ec.upload_part( + Bucket=bucket, Key=key, UploadId=upload_id, PartNumber=1, Body=data[:FIVE_MB] + ) + resp2 = s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=2, + Body=data[FIVE_MB:], + IsLastPart=True, + ) + + s3ec.complete_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={ + "Parts": [ + {"PartNumber": 1, "ETag": resp1["ETag"]}, + {"PartNumber": 2, "ETag": resp2["ETag"]}, + ] + }, + ) + except Exception: + s3ec.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + raise + + # Verify encryption metadata is present on the object + plain_s3 = boto3.client("s3") + head = plain_s3.head_object(Bucket=bucket, Key=key) + metadata = head.get("Metadata", {}) + + if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: + assert "x-amz-key-v2" in metadata + assert "x-amz-iv" in metadata + assert "x-amz-cek-alg" in metadata + assert "x-amz-wrap-alg" in metadata + else: + assert "x-amz-3" in metadata + assert "x-amz-c" in metadata + assert "x-amz-d" in metadata + assert "x-amz-i" in metadata + assert "x-amz-w" in metadata + + +# --------------------------------------------------------------------------- +# Error cases +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_part_out_of_order_fails(algorithm_suite, commitment_policy): + """Uploading parts out of sequence order must fail (serial cipher requirement).""" + key = _unique_key("mpu-ooo-") + + s3ec = _make_client(algorithm_suite, commitment_policy) + + create_resp = s3ec.create_multipart_upload(Bucket=bucket, Key=key) + upload_id = create_resp["UploadId"] + + try: + # Skip part 1, try to upload part 2 first + with pytest.raises(S3EncryptionClientError): + s3ec.upload_part( + Bucket=bucket, Key=key, UploadId=upload_id, PartNumber=2, Body=os.urandom(FIVE_MB) + ) + finally: + s3ec.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_part_invalid_upload_id_fails(algorithm_suite, commitment_policy): + """upload_part with an unknown upload ID must fail.""" + key = _unique_key("mpu-bad-id-") + + s3ec = _make_client(algorithm_suite, commitment_policy) + + with pytest.raises(S3EncryptionClientError): + s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId="nonexistent-upload-id", + PartNumber=1, + Body=os.urandom(1024), + ) + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_complete_without_parts_fails(algorithm_suite, commitment_policy): + """Completing a multipart upload without marking a final part must fail.""" + key = _unique_key("mpu-no-parts-") + + s3ec = _make_client(algorithm_suite, commitment_policy) + + create_resp = s3ec.create_multipart_upload(Bucket=bucket, Key=key) + upload_id = create_resp["UploadId"] + + try: + with pytest.raises(S3EncryptionClientError, match="final part has not been uploaded"): + s3ec.complete_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={"Parts": []}, + ) + finally: + # Clean up in case complete didn't actually fail at the S3 level + try: + s3ec.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + except Exception: + pass + + +# --------------------------------------------------------------------------- +# User metadata preservation with multipart +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_multipart_user_metadata_preserved(algorithm_suite, commitment_policy): + """User-provided metadata on create_multipart_upload should be preserved.""" + key = _unique_key("mpu-user-meta-") + user_metadata = {"author": "test-user", "version": "2.0"} + data = os.urandom(FIVE_MB + 512) + + s3ec = _make_client(algorithm_suite, commitment_policy) + + create_resp = s3ec.create_multipart_upload(Bucket=bucket, Key=key, Metadata=user_metadata) + upload_id = create_resp["UploadId"] + + try: + resp1 = s3ec.upload_part( + Bucket=bucket, Key=key, UploadId=upload_id, PartNumber=1, Body=data[:FIVE_MB] + ) + resp2 = s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=2, + Body=data[FIVE_MB:], + IsLastPart=True, + ) + + s3ec.complete_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={ + "Parts": [ + {"PartNumber": 1, "ETag": resp1["ETag"]}, + {"PartNumber": 2, "ETag": resp2["ETag"]}, + ] + }, + ) + except Exception: + s3ec.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + raise + + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == data + + returned_metadata = response.get("Metadata", {}) + for k, v in user_metadata.items(): + assert returned_metadata.get(k) == v + + +# --------------------------------------------------------------------------- +# Upload part after final part +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_part_after_final_part_fails(algorithm_suite, commitment_policy): + """Uploading a part after IsLastPart=True must fail.""" + key = _unique_key("mpu-after-final-") + + s3ec = _make_client(algorithm_suite, commitment_policy) + + create_resp = s3ec.create_multipart_upload(Bucket=bucket, Key=key) + upload_id = create_resp["UploadId"] + + try: + s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=1, + Body=os.urandom(FIVE_MB), + IsLastPart=True, + ) + + with pytest.raises(S3EncryptionClientError): + s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=2, + Body=os.urandom(1024), + ) + finally: + s3ec.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + + +# --------------------------------------------------------------------------- +# Empty body multipart +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_multipart_empty_final_part(algorithm_suite, commitment_policy): + """A multipart upload where the last part has an empty body should still work.""" + key = _unique_key("mpu-empty-last-") + part1_data = os.urandom(FIVE_MB) + + s3ec = _make_client(algorithm_suite, commitment_policy) + + create_resp = s3ec.create_multipart_upload(Bucket=bucket, Key=key) + upload_id = create_resp["UploadId"] + + try: + resp1 = s3ec.upload_part( + Bucket=bucket, Key=key, UploadId=upload_id, PartNumber=1, Body=part1_data + ) + resp2 = s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=2, + Body=b"", + IsLastPart=True, + ) + + s3ec.complete_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={ + "Parts": [ + {"PartNumber": 1, "ETag": resp1["ETag"]}, + {"PartNumber": 2, "ETag": resp2["ETag"]}, + ] + }, + ) + except Exception: + s3ec.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + raise + + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == part1_data + + +# --------------------------------------------------------------------------- +# Many parts (stress sequential cipher) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_multipart_many_parts(algorithm_suite, commitment_policy): + """Multipart upload with 10+ parts to stress the sequential cipher.""" + key = _unique_key("mpu-many-parts-") + num_parts = 12 + parts_data = [os.urandom(FIVE_MB) for _ in range(num_parts - 1)] + parts_data.append(os.urandom(1024)) # small last part + expected = b"".join(parts_data) + + s3ec = _make_client(algorithm_suite, commitment_policy) + + create_resp = s3ec.create_multipart_upload(Bucket=bucket, Key=key) + upload_id = create_resp["UploadId"] + + try: + parts = [] + for i, part_data in enumerate(parts_data, start=1): + is_last = i == num_parts + resp = s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=i, + Body=part_data, + IsLastPart=is_last, + ) + parts.append({"PartNumber": i, "ETag": resp["ETag"]}) + + s3ec.complete_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={"Parts": parts}, + ) + except Exception: + s3ec.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + raise + + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == expected + + +# --------------------------------------------------------------------------- +# Non-ASCII encryption context rejected on multipart +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_multipart_non_ascii_encryption_context_rejected(algorithm_suite, commitment_policy): + """Non-ASCII encryption context must be rejected on create_multipart_upload.""" + key = _unique_key("mpu-non-ascii-ec-") + non_ascii_contexts = [ + {"department": "ingeniería"}, + {"部門": "engineering"}, + {"emoji": "🔑"}, + ] + + s3ec = _make_client(algorithm_suite, commitment_policy) + + for ec in non_ascii_contexts: + with pytest.raises(S3EncryptionClientError, match="US-ASCII"): + s3ec.create_multipart_upload(Bucket=bucket, Key=key, EncryptionContext=ec) + + +# --------------------------------------------------------------------------- +# Caller metadata dict not mutated +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_multipart_caller_metadata_not_mutated(algorithm_suite, commitment_policy): + """create_multipart_upload must not mutate the caller's Metadata dict.""" + key = _unique_key("mpu-no-mutate-") + caller_metadata = {"author": "test"} + original_keys = set(caller_metadata.keys()) + + s3ec = _make_client(algorithm_suite, commitment_policy) + + create_resp = s3ec.create_multipart_upload(Bucket=bucket, Key=key, Metadata=caller_metadata) + upload_id = create_resp["UploadId"] + + # Clean up + s3ec.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + + assert set(caller_metadata.keys()) == original_keys + + +# --------------------------------------------------------------------------- +# Per-upload lock does not block independent uploads +# --------------------------------------------------------------------------- + + +def test_per_upload_lock_independent_uploads(): + """Per-upload locks must not block concurrent uploads to different objects.""" + s3ec = _make_client( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + + barrier = threading.Barrier(2) + results = {} + errors = [] + + def do_upload(thread_id): + try: + key = _unique_key(f"mpu-lock-{thread_id}-") + data = os.urandom(FIVE_MB + 512) + + create_resp = s3ec.create_multipart_upload(Bucket=bucket, Key=key) + upload_id = create_resp["UploadId"] + + try: + # Sync so both threads call upload_part simultaneously + barrier.wait(timeout=30) + + resp1 = s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=1, + Body=data[:FIVE_MB], + ) + + barrier.wait(timeout=30) + + resp2 = s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=2, + Body=data[FIVE_MB:], + IsLastPart=True, + ) + + s3ec.complete_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={ + "Parts": [ + {"PartNumber": 1, "ETag": resp1["ETag"]}, + {"PartNumber": 2, "ETag": resp2["ETag"]}, + ] + }, + ) + except Exception: + s3ec.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + raise + + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == data + results[thread_id] = True + + except Exception as e: + errors.append(f"Thread {thread_id}: {e}") + + t1 = threading.Thread(target=do_upload, args=(0,)) + t2 = threading.Thread(target=do_upload, args=(1,)) + t1.start() + t2.start() + t1.join(timeout=120) + t2.join(timeout=120) + + if errors: + raise AssertionError( + "Per-upload lock test failed:\n" + "\n".join(f" - {e}" for e in errors) + ) + assert len(results) == 2 + + +# --------------------------------------------------------------------------- +# Extra kwargs forwarded through upload_part +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_part_forwards_expected_bucket_owner(algorithm_suite, commitment_policy): + """upload_part must forward ExpectedBucketOwner to S3 without error.""" + key = _unique_key("mpu-fwd-kwargs-") + data = os.urandom(FIVE_MB + 512) + + s3ec = _make_client(algorithm_suite, commitment_policy) + + # Get the account ID that owns the bucket (same account we're authed as) + sts = boto3.client("sts") + account_id = sts.get_caller_identity()["Account"] + + create_resp = s3ec.create_multipart_upload(Bucket=bucket, Key=key) + upload_id = create_resp["UploadId"] + + try: + resp1 = s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=1, + Body=data[:FIVE_MB], + ExpectedBucketOwner=account_id, + ) + resp2 = s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=2, + Body=data[FIVE_MB:], + IsLastPart=True, + ExpectedBucketOwner=account_id, + ) + + s3ec.complete_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={ + "Parts": [ + {"PartNumber": 1, "ETag": resp1["ETag"]}, + {"PartNumber": 2, "ETag": resp2["ETag"]}, + ] + }, + ) + except Exception: + s3ec.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + raise + + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == data + + +# --------------------------------------------------------------------------- +# Complete failure preserves state for retry +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_complete_retryable_after_failure(algorithm_suite, commitment_policy): + """If complete_multipart_upload fails, the upload can be retried.""" + key = _unique_key("mpu-retry-complete-") + data = os.urandom(FIVE_MB + 512) + + s3ec = _make_client(algorithm_suite, commitment_policy) + + create_resp = s3ec.create_multipart_upload(Bucket=bucket, Key=key) + upload_id = create_resp["UploadId"] + + try: + resp1 = s3ec.upload_part( + Bucket=bucket, Key=key, UploadId=upload_id, PartNumber=1, Body=data[:FIVE_MB] + ) + resp2 = s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=2, + Body=data[FIVE_MB:], + IsLastPart=True, + ) + + parts = [ + {"PartNumber": 1, "ETag": resp1["ETag"]}, + {"PartNumber": 2, "ETag": resp2["ETag"]}, + ] + + # First attempt: deliberately pass bad parts to trigger S3 error + try: + s3ec.complete_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={"Parts": [{"PartNumber": 99, "ETag": '"bogus"'}]}, + ) + except S3EncryptionClientError: + pass # Expected failure + + # Retry with correct parts should succeed (state preserved) + s3ec.complete_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={"Parts": parts}, + ) + except Exception: + s3ec.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + raise + + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == data + + +# --------------------------------------------------------------------------- +# Retry upload_part with same part number +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_part_retry_same_part_number(algorithm_suite, commitment_policy): + """Calling upload_part twice with the same part number returns cached ciphertext and decrypts.""" + key = _unique_key("mpu-retry-part-") + part1_data = os.urandom(FIVE_MB) + part2_data = os.urandom(1024) + expected = part1_data + part2_data + + s3ec = _make_client(algorithm_suite, commitment_policy) + + create_resp = s3ec.create_multipart_upload(Bucket=bucket, Key=key) + upload_id = create_resp["UploadId"] + + try: + # Upload part 1 twice (simulating a retry after transient failure) + resp1_first = s3ec.upload_part( + Bucket=bucket, Key=key, UploadId=upload_id, PartNumber=1, Body=part1_data + ) + resp1_retry = s3ec.upload_part( + Bucket=bucket, Key=key, UploadId=upload_id, PartNumber=1, Body=part1_data + ) + # Both should produce the same ETag (same ciphertext uploaded) + assert resp1_first["ETag"] == resp1_retry["ETag"] + + resp2 = s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=2, + Body=part2_data, + IsLastPart=True, + ) + + s3ec.complete_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={ + "Parts": [ + {"PartNumber": 1, "ETag": resp1_retry["ETag"]}, + {"PartNumber": 2, "ETag": resp2["ETag"]}, + ] + }, + ) + except Exception: + s3ec.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + raise + + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == expected diff --git a/test/integration/test_i_s3_encryption_multithreaded.py b/test/integration/test_i_s3_encryption_multithreaded.py index e71a17df..8f713ac5 100644 --- a/test/integration/test_i_s3_encryption_multithreaded.py +++ b/test/integration/test_i_s3_encryption_multithreaded.py @@ -8,6 +8,7 @@ """ import os +import threading from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime @@ -314,3 +315,108 @@ def worker_without_context(thread_id): print("Success! Mixed threads (with and without encryption context) completed successfully.") print("Thread-local storage properly isolated context between threads.") + + +def test_concurrent_multipart_uploads(): + """Test that multiple multipart uploads can run concurrently on the same client. + + Uses a barrier to ensure upload_part calls for different objects are + interleaved, exercising the per-upload cipher isolation under contention. + """ + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig( + keyring, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + s3ec = S3EncryptionClient(wrapped_client, config) + + num_uploads = 5 + five_mb = 5 * 1024 * 1024 + errors = [] + + # Barrier ensures all threads hit upload_part at roughly the same time + barrier = threading.Barrier(num_uploads) + + def multipart_worker(thread_id): + """Create upload, sync at barrier, then upload parts interleaved with other threads.""" + try: + key = f"concurrent-mpu-{thread_id}-{datetime.now().strftime('%Y%m%d-%H%M%S-%f')}" + part1_data = os.urandom(five_mb) + part2_data = os.urandom(1024) + expected = part1_data + part2_data + + create_resp = s3ec.create_multipart_upload(Bucket=bucket, Key=key) + upload_id = create_resp["UploadId"] + + try: + # All threads wait here, then upload_part calls interleave + barrier.wait(timeout=30) + + resp1 = s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=1, + Body=part1_data, + ) + + # Second barrier to interleave part 2 as well + barrier.wait(timeout=30) + + resp2 = s3ec.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=2, + Body=part2_data, + IsLastPart=True, + ) + + s3ec.complete_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={ + "Parts": [ + {"PartNumber": 1, "ETag": resp1["ETag"]}, + {"PartNumber": 2, "ETag": resp2["ETag"]}, + ] + }, + ) + except Exception: + s3ec.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + raise + + response = s3ec.get_object(Bucket=bucket, Key=key) + decrypted = response["Body"].read() + + if decrypted != expected: + return { + "thread_id": thread_id, + "success": False, + "error": f"Data mismatch: expected {len(expected)} bytes, got {len(decrypted)}", + } + + return {"thread_id": thread_id, "success": True} + + except Exception as e: + return {"thread_id": thread_id, "success": False, "error": str(e)} + + with ThreadPoolExecutor(max_workers=num_uploads) as executor: + futures = [executor.submit(multipart_worker, i) for i in range(num_uploads)] + + for future in as_completed(futures): + result = future.result() + if not result["success"]: + errors.append( + f"Thread {result['thread_id']}: {result.get('error', 'Unknown error')}" + ) + + if errors: + raise RuntimeError( + f"{len(errors)} concurrent multipart upload(s) failed:\n" + + "\n".join(f" - {e}" for e in errors) + ) diff --git a/test/integration/test_i_s3_encryption_transfer_manager.py b/test/integration/test_i_s3_encryption_transfer_manager.py new file mode 100644 index 00000000..54ebc10b --- /dev/null +++ b/test/integration/test_i_s3_encryption_transfer_manager.py @@ -0,0 +1,396 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Integration tests for S3EncryptionClient with boto3's S3Transfer / upload_file. + +These tests verify that the S3EncryptionClient's upload_file and upload_fileobj +methods correctly handle the multipart threshold boundary, produce objects +decryptable by get_object, and behave correctly with various TransferConfig-like +parameters. + +boto3's native upload_file (via s3transfer) calls create_multipart_upload, +upload_part, and complete_multipart_upload directly on the client it wraps. +Since those calls would bypass encryption if made on the raw S3 client, +the S3EncryptionClient provides its own upload_file / upload_fileobj that +route through the encrypted multipart pipeline. +""" + +import io +import os +import tempfile +from datetime import datetime + +import boto3 +import pytest + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy + +bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") +region = os.environ.get("CI_AWS_REGION", "us-west-2") +kms_key_id = os.environ.get( + "CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" +) + +ALGORITHM_CONFIGS = [ + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + id="AES_GCM", + ), + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + id="KC_GCM", + ), +] + +ONE_MB = 1024 * 1024 + + +def _make_client(algorithm_suite, commitment_policy, **extra_config): + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig( + keyring, + encryption_algorithm=algorithm_suite, + commitment_policy=commitment_policy, + **extra_config, + ) + return S3EncryptionClient(wrapped_client, config) + + +def _unique_key(prefix): + return prefix + datetime.now().strftime("%Y-%m-%d-%H:%M:%S-%f") + + +def _write_temp_file(data): + """Write data to a temp file and return the path.""" + f = tempfile.NamedTemporaryFile(delete=False) + f.write(data) + f.close() + return f.name + + +# --------------------------------------------------------------------------- +# upload_file: below threshold → put_object path +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_file_below_threshold(algorithm_suite, commitment_policy): + """Files smaller than the threshold should use put_object internally.""" + key = _unique_key("tm-below-") + data = os.urandom(1024) + tmp = _write_temp_file(data) + + try: + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.upload_file(tmp, bucket, key) + assert s3ec.get_object(Bucket=bucket, Key=key)["Body"].read() == data + finally: + os.unlink(tmp) + + +# --------------------------------------------------------------------------- +# upload_file: above threshold → multipart path +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_file_above_default_threshold(algorithm_suite, commitment_policy): + """Files larger than the default 8 MB threshold trigger multipart upload.""" + key = _unique_key("tm-above-default-") + data = os.urandom(9 * ONE_MB) + tmp = _write_temp_file(data) + + try: + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.upload_file(tmp, bucket, key) + assert s3ec.get_object(Bucket=bucket, Key=key)["Body"].read() == data + finally: + os.unlink(tmp) + + +# --------------------------------------------------------------------------- +# upload_file: custom threshold +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_file_custom_threshold(algorithm_suite, commitment_policy): + """A custom multipart_threshold forces multipart for smaller files.""" + key = _unique_key("tm-custom-thresh-") + # 6 MB file with a 5 MB threshold → multipart + data = os.urandom(6 * ONE_MB) + tmp = _write_temp_file(data) + + try: + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.upload_file(tmp, bucket, key, multipart_threshold=5 * ONE_MB) + assert s3ec.get_object(Bucket=bucket, Key=key)["Body"].read() == data + finally: + os.unlink(tmp) + + +# --------------------------------------------------------------------------- +# upload_file: custom chunksize +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_file_custom_chunksize(algorithm_suite, commitment_policy): + """A custom multipart_chunksize controls part size (more parts).""" + key = _unique_key("tm-custom-chunk-") + # 11 MB file with 5 MB chunks → 3 parts (5 + 5 + 1) + data = os.urandom(11 * ONE_MB) + tmp = _write_temp_file(data) + + try: + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.upload_file( + tmp, + bucket, + key, + multipart_threshold=5 * ONE_MB, + multipart_chunksize=5 * ONE_MB, + ) + assert s3ec.get_object(Bucket=bucket, Key=key)["Body"].read() == data + finally: + os.unlink(tmp) + + +# --------------------------------------------------------------------------- +# upload_file: exactly at threshold boundary +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_file_exactly_at_threshold(algorithm_suite, commitment_policy): + """A file exactly equal to the threshold should use put_object (< not <=).""" + key = _unique_key("tm-exact-thresh-") + threshold = 5 * ONE_MB + data = os.urandom(threshold) + tmp = _write_temp_file(data) + + try: + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.upload_file(tmp, bucket, key, multipart_threshold=threshold) + assert s3ec.get_object(Bucket=bucket, Key=key)["Body"].read() == data + finally: + os.unlink(tmp) + + +# --------------------------------------------------------------------------- +# upload_fileobj: basic round-trip +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_fileobj_roundtrip(algorithm_suite, commitment_policy): + """upload_fileobj encrypts a BytesIO via multipart and decrypts correctly.""" + key = _unique_key("tm-fileobj-") + data = os.urandom(9 * ONE_MB) + + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.upload_fileobj(io.BytesIO(data), bucket, key) + assert s3ec.get_object(Bucket=bucket, Key=key)["Body"].read() == data + + +# --------------------------------------------------------------------------- +# upload_fileobj: small object (single part) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_fileobj_small(algorithm_suite, commitment_policy): + """upload_fileobj with a small object still works (single multipart part).""" + key = _unique_key("tm-fileobj-small-") + data = os.urandom(1024) + + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.upload_fileobj(io.BytesIO(data), bucket, key) + assert s3ec.get_object(Bucket=bucket, Key=key)["Body"].read() == data + + +# --------------------------------------------------------------------------- +# upload_file with encryption context +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_file_with_encryption_context(algorithm_suite, commitment_policy): + """upload_file passes EncryptionContext through to the multipart pipeline.""" + key = _unique_key("tm-ec-") + data = os.urandom(9 * ONE_MB) + ec = {"purpose": "transfer-manager-test"} + tmp = _write_temp_file(data) + + try: + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.upload_file(tmp, bucket, key, EncryptionContext=ec) + assert s3ec.get_object(Bucket=bucket, Key=key, EncryptionContext=ec)["Body"].read() == data + finally: + os.unlink(tmp) + + +# --------------------------------------------------------------------------- +# upload_file with user metadata +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_file_with_user_metadata(algorithm_suite, commitment_policy): + """User-provided Metadata is preserved through upload_file multipart path.""" + key = _unique_key("tm-meta-") + data = os.urandom(9 * ONE_MB) + user_meta = {"author": "test", "version": "3"} + tmp = _write_temp_file(data) + + try: + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.upload_file(tmp, bucket, key, Metadata=user_meta) + + response = s3ec.get_object(Bucket=bucket, Key=key) + assert response["Body"].read() == data + returned = response.get("Metadata", {}) + for k, v in user_meta.items(): + assert returned.get(k) == v + finally: + os.unlink(tmp) + + +# --------------------------------------------------------------------------- +# Delayed-auth decryption of transfer-manager-uploaded objects +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_file_decrypt_delayed_auth(algorithm_suite, commitment_policy): + """Objects uploaded via upload_file are decryptable in delayed-auth mode.""" + key = _unique_key("tm-delayed-") + data = os.urandom(9 * ONE_MB) + tmp = _write_temp_file(data) + + try: + writer = _make_client(algorithm_suite, commitment_policy) + writer.upload_file(tmp, bucket, key) + + reader = _make_client( + algorithm_suite, commitment_policy, enable_delayed_authentication=True + ) + response = reader.get_object(Bucket=bucket, Key=key) + result = b"" + while chunk := response["Body"].read(65536): + result += chunk + assert result == data + finally: + os.unlink(tmp) + + +# --------------------------------------------------------------------------- +# Parameter validation: zero/negative threshold and chunksize +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_file_zero_threshold_raises(algorithm_suite, commitment_policy, tmp_path): + """upload_file with multipart_threshold=0 must raise.""" + s3ec = _make_client(algorithm_suite, commitment_policy) + f = tmp_path / "test.bin" + f.write_bytes(os.urandom(1024)) + + with pytest.raises(S3EncryptionClientError, match="multipart_threshold must be a positive"): + s3ec.upload_file(str(f), bucket, "unused-key", multipart_threshold=0) + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_file_zero_chunksize_raises(algorithm_suite, commitment_policy, tmp_path): + """upload_file with multipart_chunksize=0 must raise.""" + s3ec = _make_client(algorithm_suite, commitment_policy) + f = tmp_path / "test.bin" + f.write_bytes(os.urandom(1024)) + + with pytest.raises(S3EncryptionClientError, match="multipart_chunksize must be a positive"): + s3ec.upload_file(str(f), bucket, "unused-key", multipart_chunksize=0) + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_fileobj_zero_chunksize_raises(algorithm_suite, commitment_policy): + """upload_fileobj with multipart_chunksize=0 must raise.""" + s3ec = _make_client(algorithm_suite, commitment_policy) + + with pytest.raises(S3EncryptionClientError, match="multipart_chunksize must be a positive"): + s3ec.upload_fileobj(io.BytesIO(b"data"), bucket, "unused-key", multipart_chunksize=0) + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_file_chunksize_below_5mb_raises(algorithm_suite, commitment_policy, tmp_path): + """upload_file with chunksize below S3's 5 MB minimum must raise.""" + s3ec = _make_client(algorithm_suite, commitment_policy) + f = tmp_path / "test.bin" + f.write_bytes(os.urandom(1024)) + + with pytest.raises(S3EncryptionClientError, match="at least.*5 MB"): + s3ec.upload_file(str(f), bucket, "unused-key", multipart_chunksize=1024 * 1024) + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_fileobj_chunksize_below_5mb_raises(algorithm_suite, commitment_policy): + """upload_fileobj with chunksize below S3's 5 MB minimum must raise.""" + s3ec = _make_client(algorithm_suite, commitment_policy) + + with pytest.raises(S3EncryptionClientError, match="at least.*5 MB"): + s3ec.upload_fileobj( + io.BytesIO(b"data"), bucket, "unused-key", multipart_chunksize=4 * ONE_MB + ) + + +# --------------------------------------------------------------------------- +# S3 parameters forwarded through upload_file to create_multipart_upload +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_file_forwards_content_type(algorithm_suite, commitment_policy, tmp_path): + """upload_file must forward ContentType to the multipart upload.""" + key = _unique_key("tm-content-type-") + data = os.urandom(9 * ONE_MB) + tmp = _write_temp_file(data) + + try: + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.upload_file(tmp, bucket, key, ContentType="application/octet-stream") + + # Verify ContentType was set on the object + plain_s3 = boto3.client("s3") + head = plain_s3.head_object(Bucket=bucket, Key=key) + assert head["ContentType"] == "application/octet-stream" + + # Verify data round-trips + assert s3ec.get_object(Bucket=bucket, Key=key)["Body"].read() == data + finally: + os.unlink(tmp) + + +# --------------------------------------------------------------------------- +# upload_fileobj does not close the caller's file object +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_upload_fileobj_does_not_close_caller_stream(algorithm_suite, commitment_policy): + """upload_fileobj must not close the caller's file-like object.""" + key = _unique_key("tm-no-close-") + data = os.urandom(9 * ONE_MB) + buf = io.BytesIO(data) + + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.upload_fileobj(buf, bucket, key) + + assert not buf.closed + + # Verify the upload worked + assert s3ec.get_object(Bucket=bucket, Key=key)["Body"].read() == data diff --git a/test/test_multipart.py b/test/test_multipart.py new file mode 100644 index 00000000..ca14149c --- /dev/null +++ b/test/test_multipart.py @@ -0,0 +1,780 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for multipart upload encryption pipeline and client methods.""" + +import io +import os +import threading +from unittest.mock import MagicMock + +import pytest + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager +from s3_encryption.materials.encrypted_data_key import EncryptedDataKey +from s3_encryption.materials.materials import ( + AlgorithmSuite, + CommitmentPolicy, +) +from s3_encryption.pipelines import MultipartUploadPipeline + + +def _mock_keyring(): + """Create a mock keyring that returns a fixed data key.""" + key = os.urandom(32) + keyring = MagicMock() + + def on_encrypt(mats): + + mats.plaintext_data_key = key + mats.encrypted_data_key = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms+context", + encrypted_data_key=os.urandom(64), + ) + return mats + + keyring.on_encrypt = on_encrypt + return keyring, key + + +def _make_client(algorithm_suite=None, commitment_policy=None): + """Create an S3EncryptionClient with a mock keyring and mock S3 client.""" + keyring, _ = _mock_keyring() + algo = algorithm_suite or AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + policy = commitment_policy or CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + config = S3EncryptionClientConfig( + keyring=keyring, + encryption_algorithm=algo, + commitment_policy=policy, + ) + mock_s3 = MagicMock() + mock_s3.meta.config.user_agent_extra = "" + mock_s3.meta.events = MagicMock() + return S3EncryptionClient(mock_s3, config) + + +class TestMultipartUploadPipeline: + """Unit tests for the MultipartUploadPipeline cipher logic.""" + + @pytest.fixture( + params=[ + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ] + ) + def pipeline(self, request): + + keyring, _ = _mock_keyring() + cmm = DefaultCryptoMaterialsManager(keyring) + return MultipartUploadPipeline( + cmm=cmm, + encryption_algorithm=request.param, + ) + + def test_encrypt_single_part(self, pipeline): + data = b"hello world" + ct = pipeline.encrypt_part(1, data, is_last=True) + # Ciphertext should be data + 16-byte GCM tag + assert len(ct) == len(data) + 16 + assert pipeline.has_final_part_been_seen + + def test_encrypt_multiple_parts(self, pipeline): + part1 = pipeline.encrypt_part(1, b"A" * 1024) + part2 = pipeline.encrypt_part(2, b"B" * 512, is_last=True) + assert len(part1) == 1024 + assert len(part2) == 512 + 16 # data + tag on last part + assert pipeline.has_final_part_been_seen + + def test_out_of_order_raises(self, pipeline): + with pytest.raises(S3EncryptionClientError, match="sequence"): + pipeline.encrypt_part(2, b"data") + + def test_part_after_final_raises(self, pipeline): + pipeline.encrypt_part(1, b"data", is_last=True) + with pytest.raises(S3EncryptionClientError, match="after the final part"): + pipeline.encrypt_part(2, b"more data") + + def test_empty_part(self, pipeline): + ct = pipeline.encrypt_part(1, b"", is_last=True) + # Empty data + 16-byte tag + assert len(ct) == 16 + + def test_metadata_present(self, pipeline): + assert pipeline.metadata + # Should have encryption metadata keys + assert len(pipeline.metadata) > 0 + + def test_string_body_converted(self, pipeline): + ct = pipeline.encrypt_part(1, "hello", is_last=True) + assert len(ct) == len(b"hello") + 16 + + +class TestS3EncryptionClientMultipart: + """Unit tests for the S3EncryptionClient multipart methods.""" + + def test_create_multipart_upload(self): + s3ec = _make_client() + s3ec.wrapped_s3_client.create_multipart_upload.return_value = { + "UploadId": "test-upload-id", + "Bucket": "bucket", + "Key": "key", + } + + resp = s3ec.create_multipart_upload(Bucket="bucket", Key="key") + assert resp["UploadId"] == "test-upload-id" + s3ec.wrapped_s3_client.create_multipart_upload.assert_called_once() + + def test_upload_part_unknown_upload_id(self): + s3ec = _make_client() + with pytest.raises(S3EncryptionClientError, match="No multipart upload found"): + s3ec.upload_part( + Bucket="bucket", Key="key", UploadId="nonexistent", PartNumber=1, Body=b"data" + ) + + def test_upload_part_encrypts(self): + s3ec = _make_client() + s3ec.wrapped_s3_client.create_multipart_upload.return_value = { + "UploadId": "uid-1", + "Bucket": "bucket", + "Key": "key", + } + s3ec.wrapped_s3_client.upload_part.return_value = {"ETag": '"abc123"'} + + s3ec.create_multipart_upload(Bucket="bucket", Key="key") + resp = s3ec.upload_part( + Bucket="bucket", + Key="key", + UploadId="uid-1", + PartNumber=1, + Body=b"data", + IsLastPart=True, + ) + + assert resp["ETag"] == '"abc123"' + # Verify the body passed to S3 is ciphertext (different from plaintext) + call_kwargs = s3ec.wrapped_s3_client.upload_part.call_args[1] + assert call_kwargs["Body"] != b"data" + + def test_complete_without_final_part_raises(self): + s3ec = _make_client() + s3ec.wrapped_s3_client.create_multipart_upload.return_value = { + "UploadId": "uid-2", + "Bucket": "bucket", + "Key": "key", + } + + s3ec.create_multipart_upload(Bucket="bucket", Key="key") + + with pytest.raises(S3EncryptionClientError, match="final part has not been uploaded"): + s3ec.complete_multipart_upload( + Bucket="bucket", + Key="key", + UploadId="uid-2", + MultipartUpload={"Parts": []}, + ) + + def test_complete_after_final_part_succeeds(self): + s3ec = _make_client() + s3ec.wrapped_s3_client.create_multipart_upload.return_value = { + "UploadId": "uid-3", + "Bucket": "bucket", + "Key": "key", + } + s3ec.wrapped_s3_client.upload_part.return_value = {"ETag": '"etag1"'} + s3ec.wrapped_s3_client.complete_multipart_upload.return_value = {"Location": "s3://..."} + + s3ec.create_multipart_upload(Bucket="bucket", Key="key") + s3ec.upload_part( + Bucket="bucket", + Key="key", + UploadId="uid-3", + PartNumber=1, + Body=b"x" * 1024, + IsLastPart=True, + ) + resp = s3ec.complete_multipart_upload( + Bucket="bucket", + Key="key", + UploadId="uid-3", + MultipartUpload={"Parts": [{"PartNumber": 1, "ETag": '"etag1"'}]}, + ) + assert resp["Location"] == "s3://..." + + def test_abort_cleans_up_state(self): + s3ec = _make_client() + s3ec.wrapped_s3_client.create_multipart_upload.return_value = { + "UploadId": "uid-4", + "Bucket": "bucket", + "Key": "key", + } + s3ec.wrapped_s3_client.abort_multipart_upload.return_value = {} + + s3ec.create_multipart_upload(Bucket="bucket", Key="key") + s3ec.abort_multipart_upload(Bucket="bucket", Key="key", UploadId="uid-4") + + # After abort, upload_part should fail + with pytest.raises(S3EncryptionClientError, match="No multipart upload found"): + s3ec.upload_part( + Bucket="bucket", Key="key", UploadId="uid-4", PartNumber=1, Body=b"data" + ) + + def test_complete_unknown_upload_id_raises(self): + s3ec = _make_client() + with pytest.raises(S3EncryptionClientError, match="No multipart upload found"): + s3ec.complete_multipart_upload( + Bucket="bucket", + Key="key", + UploadId="nonexistent", + MultipartUpload={"Parts": []}, + ) + + def test_create_multipart_with_encryption_context(self): + s3ec = _make_client() + s3ec.wrapped_s3_client.create_multipart_upload.return_value = { + "UploadId": "uid-ec", + "Bucket": "bucket", + "Key": "key", + } + + s3ec.create_multipart_upload(Bucket="bucket", Key="key", EncryptionContext={"env": "test"}) + + # EncryptionContext should not be passed to S3 (it's consumed by the pipeline) + call_kwargs = s3ec.wrapped_s3_client.create_multipart_upload.call_args[1] + assert "EncryptionContext" not in call_kwargs + + def test_metadata_merged_on_create(self): + s3ec = _make_client() + s3ec.wrapped_s3_client.create_multipart_upload.return_value = { + "UploadId": "uid-meta", + "Bucket": "bucket", + "Key": "key", + } + + s3ec.create_multipart_upload( + Bucket="bucket", Key="key", Metadata={"user-key": "user-value"} + ) + + call_kwargs = s3ec.wrapped_s3_client.create_multipart_upload.call_args[1] + metadata = call_kwargs["Metadata"] + # User metadata preserved + assert metadata["user-key"] == "user-value" + # Encryption metadata also present + assert len(metadata) > 1 + + +class TestUploadFileAndFileobj: + """Unit tests for upload_file and upload_fileobj high-level methods.""" + + def _setup_client(self): + s3ec = _make_client() + s3ec.wrapped_s3_client.create_multipart_upload.return_value = { + "UploadId": "uid-file", + "Bucket": "bucket", + "Key": "key", + } + s3ec.wrapped_s3_client.upload_part.return_value = {"ETag": '"etag"'} + s3ec.wrapped_s3_client.complete_multipart_upload.return_value = {"Location": "s3://..."} + return s3ec + + def test_upload_file_below_threshold_uses_put_object(self, tmp_path): + s3ec = _make_client() + # Mock put_object on the event-based path + s3ec.wrapped_s3_client.put_object.return_value = {} + + f = tmp_path / "small.bin" + f.write_bytes(b"small data") + + s3ec.upload_file(str(f), "bucket", "key", multipart_threshold=1024 * 1024) + + # put_object should have been called (via the event system) + s3ec.wrapped_s3_client.put_object.assert_called_once() + s3ec.wrapped_s3_client.create_multipart_upload.assert_not_called() + + def test_upload_file_above_threshold_uses_multipart(self, tmp_path): + s3ec = self._setup_client() + + f = tmp_path / "large.bin" + f.write_bytes(os.urandom(2048)) + + s3ec.upload_file( + str(f), "bucket", "key", multipart_threshold=1024, multipart_chunksize=5 * 1024 * 1024 + ) + + s3ec.wrapped_s3_client.create_multipart_upload.assert_called_once() + assert s3ec.wrapped_s3_client.upload_part.call_count >= 1 + s3ec.wrapped_s3_client.complete_multipart_upload.assert_called_once() + + def test_upload_fileobj_uses_multipart(self): + + s3ec = self._setup_client() + data = os.urandom(2048) + + s3ec.upload_fileobj(io.BytesIO(data), "bucket", "key", multipart_chunksize=5 * 1024 * 1024) + + s3ec.wrapped_s3_client.create_multipart_upload.assert_called_once() + assert s3ec.wrapped_s3_client.upload_part.call_count >= 1 + s3ec.wrapped_s3_client.complete_multipart_upload.assert_called_once() + + def test_upload_file_aborts_on_failure(self, tmp_path): + s3ec = _make_client() + s3ec.wrapped_s3_client.create_multipart_upload.return_value = { + "UploadId": "uid-fail", + "Bucket": "bucket", + "Key": "key", + } + s3ec.wrapped_s3_client.upload_part.side_effect = Exception("network error") + s3ec.wrapped_s3_client.abort_multipart_upload.return_value = {} + + f = tmp_path / "fail.bin" + f.write_bytes(os.urandom(2048)) + + with pytest.raises(Exception): + s3ec.upload_file( + str(f), + "bucket", + "key", + multipart_threshold=1024, + multipart_chunksize=5 * 1024 * 1024, + ) + + s3ec.wrapped_s3_client.abort_multipart_upload.assert_called_once() + + def test_upload_file_passes_encryption_context(self, tmp_path): + s3ec = self._setup_client() + + f = tmp_path / "ec.bin" + f.write_bytes(os.urandom(2048)) + + s3ec.upload_file( + str(f), + "bucket", + "key", + multipart_threshold=1024, + multipart_chunksize=5 * 1024 * 1024, + EncryptionContext={"env": "test"}, + ) + + # EncryptionContext consumed by create_multipart_upload, not passed to S3 + create_kwargs = s3ec.wrapped_s3_client.create_multipart_upload.call_args[1] + assert "EncryptionContext" not in create_kwargs + + def test_upload_file_passes_user_metadata(self, tmp_path): + s3ec = self._setup_client() + + f = tmp_path / "meta.bin" + f.write_bytes(os.urandom(2048)) + + s3ec.upload_file( + str(f), + "bucket", + "key", + multipart_threshold=1024, + multipart_chunksize=5 * 1024 * 1024, + Metadata={"author": "test"}, + ) + + create_kwargs = s3ec.wrapped_s3_client.create_multipart_upload.call_args[1] + assert create_kwargs["Metadata"]["author"] == "test" + + +class TestMultipartEncryptionContextValidation: + """Unit tests for encryption context validation in create_multipart_upload.""" + + def test_non_ascii_value_rejected(self): + s3ec = _make_client() + with pytest.raises(S3EncryptionClientError, match="US-ASCII"): + s3ec.create_multipart_upload( + Bucket="bucket", Key="key", EncryptionContext={"key": "válue"} + ) + + def test_non_ascii_key_rejected(self): + s3ec = _make_client() + with pytest.raises(S3EncryptionClientError, match="US-ASCII"): + s3ec.create_multipart_upload( + Bucket="bucket", Key="key", EncryptionContext={"clé": "value"} + ) + + def test_emoji_rejected(self): + s3ec = _make_client() + with pytest.raises(S3EncryptionClientError, match="US-ASCII"): + s3ec.create_multipart_upload( + Bucket="bucket", Key="key", EncryptionContext={"emoji": "🔑"} + ) + + def test_ascii_context_accepted(self): + s3ec = _make_client() + s3ec.wrapped_s3_client.create_multipart_upload.return_value = { + "UploadId": "uid-ascii", + "Bucket": "bucket", + "Key": "key", + } + # Should not raise + resp = s3ec.create_multipart_upload( + Bucket="bucket", Key="key", EncryptionContext={"env": "test"} + ) + assert resp["UploadId"] == "uid-ascii" + + def test_caller_metadata_dict_not_mutated(self): + s3ec = _make_client() + s3ec.wrapped_s3_client.create_multipart_upload.return_value = { + "UploadId": "uid-nomutate", + "Bucket": "bucket", + "Key": "key", + } + + caller_metadata = {"author": "test"} + original_keys = set(caller_metadata.keys()) + + s3ec.create_multipart_upload(Bucket="bucket", Key="key", Metadata=caller_metadata) + + # Caller's dict should not have been modified with encryption metadata + assert set(caller_metadata.keys()) == original_keys + + +class TestMultipartPipelineLock: + """Unit tests verifying per-upload lock prevents concurrent encrypt_part races.""" + + def test_concurrent_encrypt_part_same_pipeline_serialized(self): + """Concurrent calls to encrypt_part on the same pipeline are serialized by the lock.""" + keyring, _ = _mock_keyring() + cmm = DefaultCryptoMaterialsManager(keyring) + pipeline = MultipartUploadPipeline( + cmm=cmm, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ) + + results = {} + errors = [] + barrier = threading.Barrier(2) + + def upload_part_1(): + try: + barrier.wait(timeout=5) + ct = pipeline.encrypt_part(1, b"A" * 1024) + results[1] = ct + except Exception as e: + errors.append(("part1", e)) + + def upload_part_2(): + try: + barrier.wait(timeout=5) + ct = pipeline.encrypt_part(2, b"B" * 512, is_last=True) + results[2] = ct + except Exception as e: + errors.append(("part2", e)) + + t1 = threading.Thread(target=upload_part_1) + t2 = threading.Thread(target=upload_part_2) + t1.start() + t2.start() + t1.join(timeout=10) + t2.join(timeout=10) + + # One of two outcomes is valid: + # 1. Both succeed in order (part 1 acquired lock first) + # 2. Part 2 fails with sequence error (part 2 acquired lock first) + if errors: + # If there's an error, it must be a sequence error on part 2 + assert any("sequence" in str(e).lower() for _, e in errors) + else: + # Both succeeded means part 1 ran first + assert 1 in results and 2 in results + assert len(results[1]) == 1024 + assert len(results[2]) == 512 + 16 + + def test_upload_part_forwards_extra_kwargs(self): + """upload_part must forward extra S3 parameters (e.g. RequestPayer) to the S3 client.""" + s3ec = _make_client() + s3ec.wrapped_s3_client.create_multipart_upload.return_value = { + "UploadId": "uid-fwd", + "Bucket": "bucket", + "Key": "key", + } + s3ec.wrapped_s3_client.upload_part.return_value = {"ETag": '"etag"'} + + s3ec.create_multipart_upload(Bucket="bucket", Key="key") + s3ec.upload_part( + Bucket="bucket", + Key="key", + UploadId="uid-fwd", + PartNumber=1, + Body=b"data", + IsLastPart=True, + RequestPayer="requester", + ExpectedBucketOwner="123456789012", + ) + + call_kwargs = s3ec.wrapped_s3_client.upload_part.call_args[1] + assert call_kwargs["RequestPayer"] == "requester" + assert call_kwargs["ExpectedBucketOwner"] == "123456789012" + # IsLastPart should NOT be forwarded to S3 + assert "IsLastPart" not in call_kwargs + + def test_upload_part_does_not_forward_is_last_part(self): + """IsLastPart is consumed by the client and must not reach S3.""" + s3ec = _make_client() + s3ec.wrapped_s3_client.create_multipart_upload.return_value = { + "UploadId": "uid-nolast", + "Bucket": "bucket", + "Key": "key", + } + s3ec.wrapped_s3_client.upload_part.return_value = {"ETag": '"etag"'} + + s3ec.create_multipart_upload(Bucket="bucket", Key="key") + s3ec.upload_part( + Bucket="bucket", + Key="key", + UploadId="uid-nolast", + PartNumber=1, + Body=b"x", + IsLastPart=True, + ) + + call_kwargs = s3ec.wrapped_s3_client.upload_part.call_args[1] + assert "IsLastPart" not in call_kwargs + + def test_complete_failure_preserves_state_for_retry(self): + """If complete_multipart_upload fails, the upload state is preserved for retry.""" + s3ec = _make_client() + s3ec.wrapped_s3_client.create_multipart_upload.return_value = { + "UploadId": "uid-retry", + "Bucket": "bucket", + "Key": "key", + } + s3ec.wrapped_s3_client.upload_part.return_value = {"ETag": '"etag1"'} + + s3ec.create_multipart_upload(Bucket="bucket", Key="key") + s3ec.upload_part( + Bucket="bucket", + Key="key", + UploadId="uid-retry", + PartNumber=1, + Body=b"data", + IsLastPart=True, + ) + + # First complete fails + s3ec.wrapped_s3_client.complete_multipart_upload.side_effect = Exception("network timeout") + with pytest.raises(S3EncryptionClientError, match="network timeout"): + s3ec.complete_multipart_upload( + Bucket="bucket", + Key="key", + UploadId="uid-retry", + MultipartUpload={"Parts": [{"PartNumber": 1, "ETag": '"etag1"'}]}, + ) + + # Retry should work (state not cleaned up) + s3ec.wrapped_s3_client.complete_multipart_upload.side_effect = None + s3ec.wrapped_s3_client.complete_multipart_upload.return_value = {"Location": "s3://ok"} + resp = s3ec.complete_multipart_upload( + Bucket="bucket", + Key="key", + UploadId="uid-retry", + MultipartUpload={"Parts": [{"PartNumber": 1, "ETag": '"etag1"'}]}, + ) + assert resp["Location"] == "s3://ok" + + # After success, state is cleaned up + with pytest.raises(S3EncryptionClientError, match="No multipart upload found"): + s3ec.complete_multipart_upload( + Bucket="bucket", + Key="key", + UploadId="uid-retry", + MultipartUpload={"Parts": []}, + ) + + +class TestUploadFileValidation: + """Unit tests for upload_file/upload_fileobj parameter validation.""" + + def test_zero_threshold_raises(self, tmp_path): + s3ec = _make_client() + f = tmp_path / "test.bin" + f.write_bytes(b"data") + with pytest.raises(S3EncryptionClientError, match="multipart_threshold must be a positive"): + s3ec.upload_file(str(f), "bucket", "key", multipart_threshold=0) + + def test_negative_threshold_raises(self, tmp_path): + s3ec = _make_client() + f = tmp_path / "test.bin" + f.write_bytes(b"data") + with pytest.raises(S3EncryptionClientError, match="multipart_threshold must be a positive"): + s3ec.upload_file(str(f), "bucket", "key", multipart_threshold=-1) + + def test_zero_chunksize_raises(self, tmp_path): + s3ec = _make_client() + f = tmp_path / "test.bin" + f.write_bytes(b"data") + with pytest.raises(S3EncryptionClientError, match="multipart_chunksize must be a positive"): + s3ec.upload_file(str(f), "bucket", "key", multipart_chunksize=0) + + def test_negative_chunksize_raises(self, tmp_path): + s3ec = _make_client() + f = tmp_path / "test.bin" + f.write_bytes(b"data") + with pytest.raises(S3EncryptionClientError, match="multipart_chunksize must be a positive"): + s3ec.upload_file(str(f), "bucket", "key", multipart_chunksize=-1) + + def test_upload_fileobj_zero_chunksize_raises(self): + + s3ec = _make_client() + with pytest.raises(S3EncryptionClientError, match="multipart_chunksize must be a positive"): + s3ec.upload_fileobj(io.BytesIO(b"data"), "bucket", "key", multipart_chunksize=0) + + def test_upload_fileobj_negative_chunksize_raises(self): + + s3ec = _make_client() + with pytest.raises(S3EncryptionClientError, match="multipart_chunksize must be a positive"): + s3ec.upload_fileobj(io.BytesIO(b"data"), "bucket", "key", multipart_chunksize=-1) + + def test_chunksize_below_5mb_raises(self, tmp_path): + s3ec = _make_client() + f = tmp_path / "test.bin" + f.write_bytes(os.urandom(1024)) + with pytest.raises(S3EncryptionClientError, match="at least.*5 MB"): + s3ec.upload_file(str(f), "bucket", "key", multipart_chunksize=1024 * 1024) + + def test_upload_fileobj_chunksize_below_5mb_raises(self): + + s3ec = _make_client() + with pytest.raises(S3EncryptionClientError, match="at least.*5 MB"): + s3ec.upload_fileobj( + io.BytesIO(b"data"), "bucket", "key", multipart_chunksize=4 * 1024 * 1024 + ) + + def test_upload_file_forwards_s3_params_to_create(self, tmp_path): + """upload_file must forward S3 params like ContentType to create_multipart_upload.""" + s3ec = _make_client() + s3ec.wrapped_s3_client.create_multipart_upload.return_value = { + "UploadId": "uid-fwd-create", + "Bucket": "bucket", + "Key": "key", + } + s3ec.wrapped_s3_client.upload_part.return_value = {"ETag": '"etag"'} + s3ec.wrapped_s3_client.complete_multipart_upload.return_value = {"Location": "s3://..."} + + f = tmp_path / "typed.bin" + f.write_bytes(os.urandom(2048)) + + s3ec.upload_file( + str(f), + "bucket", + "key", + multipart_threshold=1024, + multipart_chunksize=5 * 1024 * 1024, + ContentType="application/json", + Tagging="env=test", + ) + + create_kwargs = s3ec.wrapped_s3_client.create_multipart_upload.call_args[1] + assert create_kwargs["ContentType"] == "application/json" + assert create_kwargs["Tagging"] == "env=test" + + +class TestFileobjLifecycle: + """Unit tests verifying upload_fileobj does not close the caller's file object.""" + + def _setup_client(self): + s3ec = _make_client() + s3ec.wrapped_s3_client.create_multipart_upload.return_value = { + "UploadId": "uid-lifecycle", + "Bucket": "bucket", + "Key": "key", + } + s3ec.wrapped_s3_client.upload_part.return_value = {"ETag": '"etag"'} + s3ec.wrapped_s3_client.complete_multipart_upload.return_value = {"Location": "s3://..."} + return s3ec + + def test_upload_fileobj_does_not_close_caller_stream(self): + + s3ec = self._setup_client() + buf = io.BytesIO(os.urandom(1024)) + + s3ec.upload_fileobj(buf, "bucket", "key") + + assert not buf.closed + + def test_upload_file_closes_its_own_stream(self, tmp_path): + """upload_file opens the file internally and must close it after.""" + s3ec = self._setup_client() + + f = tmp_path / "owned.bin" + f.write_bytes(os.urandom(2048)) + + s3ec.upload_file( + str(f), "bucket", "key", multipart_threshold=1024, multipart_chunksize=5 * 1024 * 1024 + ) + + # We can't directly check the internal file handle is closed, + # but we can verify the upload completed without error and the + # file is still readable (not locked) + assert f.read_bytes() == f.read_bytes() + + +class TestMultipartPartRetry: + """Unit tests for retrying a failed upload_part call.""" + + @pytest.fixture + def pipeline(self): + + keyring, _ = _mock_keyring() + cmm = DefaultCryptoMaterialsManager(keyring) + return MultipartUploadPipeline( + cmm=cmm, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ) + + def test_retry_same_part_returns_cached_ciphertext(self, pipeline): + ct1 = pipeline.encrypt_part(1, b"hello") + ct2 = pipeline.encrypt_part(1, b"hello") + assert ct1 == ct2 + + def test_retry_last_part_returns_cached_ciphertext(self, pipeline): + pipeline.encrypt_part(1, b"part one") + ct2 = pipeline.encrypt_part(2, b"part two", is_last=True) + ct2_retry = pipeline.encrypt_part(2, b"part two", is_last=True) + assert ct2 == ct2_retry + + def test_retry_does_not_block_next_part(self, pipeline): + pipeline.encrypt_part(1, b"first") + # Retry part 1 + pipeline.encrypt_part(1, b"first") + # Part 2 should still work + ct = pipeline.encrypt_part(2, b"second", is_last=True) + assert len(ct) == len(b"second") + 16 + + def test_client_upload_part_retry_after_s3_failure(self): + """If S3 upload_part fails, retrying the same part number succeeds.""" + s3ec = _make_client() + s3ec.wrapped_s3_client.create_multipart_upload.return_value = { + "UploadId": "uid-retry-part", + "Bucket": "bucket", + "Key": "key", + } + + s3ec.create_multipart_upload(Bucket="bucket", Key="key") + + # First attempt fails at S3 level + s3ec.wrapped_s3_client.upload_part.side_effect = Exception("network error") + with pytest.raises(Exception, match="network error"): + s3ec.upload_part( + Bucket="bucket", + Key="key", + UploadId="uid-retry-part", + PartNumber=1, + Body=b"data", + ) + + # Retry succeeds + s3ec.wrapped_s3_client.upload_part.side_effect = None + s3ec.wrapped_s3_client.upload_part.return_value = {"ETag": '"etag1"'} + resp = s3ec.upload_part( + Bucket="bucket", + Key="key", + UploadId="uid-retry-part", + PartNumber=1, + Body=b"data", + ) + assert resp["ETag"] == '"etag1"' diff --git a/test/test_stream.py b/test/test_stream.py index 692c8b00..ffa43e1c 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -218,14 +218,20 @@ def test_enter_returns_self(self): def test_wrong_key_raises_error(self): plaintext = b"wrong key test!!" ciphertext, _key, iv = _encrypt_cbc(plaintext) - wrong_key = os.urandom(32) - stream = DecryptingStream( - _make_streaming_body(ciphertext), - _make_cbc_decryptor(wrong_key, iv, len(ciphertext)), - content_length=len(ciphertext), - ) - with pytest.raises(S3EncryptionClientSecurityError, match="Failed to decrypt CBC content"): - stream.read() + # ~1/256 chance random garbage has valid PKCS7 padding, so retry + for _ in range(10): + wrong_key = os.urandom(32) + stream = DecryptingStream( + _make_streaming_body(ciphertext), + _make_cbc_decryptor(wrong_key, iv, len(ciphertext)), + content_length=len(ciphertext), + ) + try: + stream.read() + except S3EncryptionClientSecurityError as e: + assert "Failed to decrypt CBC content" in str(e) + return + pytest.fail("Wrong key did not produce CBC decryption error after 10 attempts") def test_empty_ciphertext(self): key = os.urandom(32) From 3d133b178533cea6add980accd68229851c5d451 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Mon, 18 May 2026 16:55:19 -0700 Subject: [PATCH 80/81] chore: setup release, readthedocs (#186) --- .github/workflows/release.yml | 230 +++++++++++++++++++++++++++++++++ .gitignore | 3 + .readthedocs.yaml | 16 +++ .releaserc.cjs | 70 ++++++++++ Makefile | 11 +- docs/api.rst | 35 +++++ docs/conf.py | 38 ++++++ docs/index.rst | 41 ++++++ pyproject.toml | 4 + release-validation/validate.py | 60 +++++++++ src/s3_encryption/__init__.py | 4 +- 11 files changed, 509 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 .readthedocs.yaml create mode 100644 .releaserc.cjs create mode 100644 docs/api.rst create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 release-validation/validate.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..72742fe0 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,230 @@ +name: Release to PyPI + +on: + workflow_dispatch: + inputs: + version_override: + description: "Manual version override (leave empty to use semantic-release)" + required: false + type: string + dry_run: + description: "Dry run (determine version only, do not publish)" + required: false + type: boolean + default: false + +jobs: + determine-version: + name: Determine Version + runs-on: ubuntu-latest + permissions: + contents: write + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + persist-credentials: true + - uses: actions/setup-node@v4 + with: + node-version: "26" + - name: Install semantic-release + run: npm install -g semantic-release @semantic-release/commit-analyzer @semantic-release/release-notes-generator @semantic-release/changelog @semantic-release/exec @semantic-release/git @semantic-release/github conventional-changelog-conventionalcommits + - name: Determine next version + id: version + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [ -n "${{ inputs.version_override }}" ]; then + echo "version=${{ inputs.version_override }}" >> "$GITHUB_OUTPUT" + echo "Using manual override: ${{ inputs.version_override }}" + else + # Run semantic-release in dry-run to get the next version + VERSION=$(npx semantic-release --dry-run 2>&1 | grep -oP 'The next release version is \K[0-9]+\.[0-9]+\.[0-9]+' || true) + if [ -z "$VERSION" ]; then + echo "No release needed based on commits" + echo "version=" >> "$GITHUB_OUTPUT" + else + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "Semantic release determined version: $VERSION" + fi + fi + + test: + name: Run Tests + needs: determine-version + if: needs.determine-version.outputs.version != '' + uses: ./.github/workflows/python-integ.yml + permissions: + id-token: write + contents: read + secrets: inherit + + build: + name: Build Package + needs: [determine-version, test] + if: needs.determine-version.outputs.version != '' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.10" + - run: pip install build + - name: Set version in pyproject.toml + run: sed -i "s/^version = .*/version = \"${{ needs.determine-version.outputs.version }}\"/" pyproject.toml + - name: Verify version + run: | + grep "version = \"${{ needs.determine-version.outputs.version }}\"" pyproject.toml + - run: python -m build + - uses: actions/upload-artifact@v7 + with: + name: dist + path: dist/ + + publish-testpypi: + name: Publish to TestPyPI + if: ${{ !inputs.dry_run && needs.determine-version.outputs.version != '' }} + needs: [determine-version, build] + runs-on: ubuntu-latest + environment: testpypi + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v7 + with: + name: dist + path: dist/ + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + skip-existing: true + + validate-testpypi: + name: Validate TestPyPI Package + needs: [determine-version, publish-testpypi] + if: needs.determine-version.outputs.version != '' + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v6 + with: + sparse-checkout: release-validation + - uses: actions/setup-python@v6 + with: + python-version: "3.10" + - uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role + aws-region: us-west-2 + - name: Wait for TestPyPI availability + run: | + for i in $(seq 1 30); do + if pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ "amazon-s3-encryption-client-python==${{ needs.determine-version.outputs.version }}" 2>/dev/null; then + echo "Package available on TestPyPI" + exit 0 + fi + echo "Waiting for package to appear on TestPyPI ($i/30)..." + sleep 10 + done + echo "Package not found on TestPyPI after 5 minutes" + exit 1 + - name: Run validation + run: python release-validation/validate.py + env: + CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} + CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} + + publish-pypi: + name: Publish to PyPI + needs: [determine-version, validate-testpypi] + if: needs.determine-version.outputs.version != '' + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v7 + with: + name: dist + path: dist/ + - uses: pypa/gh-action-pypi-publish@release/v1 + + validate-pypi: + name: Validate PyPI Package + needs: [determine-version, publish-pypi] + if: needs.determine-version.outputs.version != '' + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v6 + with: + sparse-checkout: release-validation + - uses: actions/setup-python@v6 + with: + python-version: "3.10" + - uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role + aws-region: us-west-2 + - name: Wait for PyPI availability + run: | + for i in $(seq 1 30); do + if pip install "amazon-s3-encryption-client-python==${{ needs.determine-version.outputs.version }}" 2>/dev/null; then + echo "Package available on PyPI" + exit 0 + fi + echo "Waiting for package to appear on PyPI ($i/30)..." + sleep 10 + done + echo "Package not found on PyPI after 5 minutes" + exit 1 + - name: Run validation + run: python release-validation/validate.py + env: + CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} + CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} + + create-release: + name: Create GitHub Release + if: ${{ !inputs.dry_run && needs.determine-version.outputs.version != '' }} + needs: [determine-version, validate-pypi] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + persist-credentials: true + - uses: actions/setup-node@v4 + with: + node-version: "26" + - name: Install semantic-release + run: npm install -g semantic-release @semantic-release/commit-analyzer @semantic-release/release-notes-generator @semantic-release/changelog @semantic-release/exec @semantic-release/git @semantic-release/github conventional-changelog-conventionalcommits + - name: Create release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + VERSION="${{ needs.determine-version.outputs.version }}" + if [ -n "${{ inputs.version_override }}" ]; then + # Manual override: commit the version bump and create a GitHub release + sed -i "s/^version = .*/version = \"${VERSION}\"/" pyproject.toml + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add pyproject.toml + git commit -m "chore(release): ${VERSION} [skip ci]" || true + git tag "v${VERSION}" + git push --follow-tags + gh release create "v${VERSION}" \ + --title "v${VERSION}" \ + --generate-notes \ + --draft + else + npx semantic-release + fi diff --git a/.gitignore b/.gitignore index b0b67407..39fc3914 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,6 @@ smithy-java-core/out .coverage coverage-report/ perf-results/ + +# Sphinx docs build output +docs/_build/ diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..26768c7d --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,16 @@ +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +sphinx: + configuration: docs/conf.py + +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/.releaserc.cjs b/.releaserc.cjs new file mode 100644 index 00000000..bdd99b04 --- /dev/null +++ b/.releaserc.cjs @@ -0,0 +1,70 @@ +/** + * Semantic Release configuration for Amazon S3 Encryption Client for Python. + * + * Determines the next version from conventional commits, updates pyproject.toml, + * generates release notes, and creates a GitHub release. + */ +module.exports = { + branches: ["main"], + plugins: [ + [ + "@semantic-release/commit-analyzer", + { + preset: "conventionalcommits", + releaseRules: [ + { type: "feat", release: "minor" }, + { type: "fix", release: "patch" }, + { type: "perf", release: "patch" }, + { type: "revert", release: "patch" }, + { breaking: true, release: "major" }, + ], + }, + ], + [ + "@semantic-release/release-notes-generator", + { + preset: "conventionalcommits", + presetConfig: { + types: [ + { type: "feat", section: "Features" }, + { type: "fix", section: "Bug Fixes" }, + { type: "perf", section: "Performance" }, + { type: "revert", section: "Reverts" }, + { type: "docs", section: "Documentation", hidden: false }, + { type: "chore", section: "Maintenance", hidden: false }, + { type: "refactor", section: "Refactoring", hidden: false }, + { type: "test", section: "Tests", hidden: true }, + { type: "ci", section: "CI", hidden: true }, + ], + }, + }, + ], + [ + "@semantic-release/exec", + { + prepareCmd: + 'sed -i "s/^version = .*/version = \\"${nextRelease.version}\\"/" pyproject.toml', + }, + ], + [ + "@semantic-release/changelog", + { + changelogFile: "CHANGELOG.md", + }, + ], + [ + "@semantic-release/git", + { + assets: ["pyproject.toml", "CHANGELOG.md"], + message: + "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}", + }, + ], + [ + "@semantic-release/github", + { + draftRelease: true, + }, + ], + ], +}; diff --git a/Makefile b/Makefile index d01d75a3..e788379b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: lint format test test-unit test-integration test-perf install +.PHONY: lint format format-check test test-unit test-integration test-perf install docs # Default target all: lint test duvet @@ -56,3 +56,12 @@ duvet-report: duvet-view-report-mac: open .duvet/reports/report.html + +# Build docs locally +docs: + uv pip install -e ".[docs]" + uv run sphinx-build -b html docs/ docs/_build/html + @echo "Docs built at docs/_build/html/index.html" + +docs-open: docs + open docs/_build/html/index.html diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 00000000..4611623b --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,35 @@ +API Reference +============= + +Client +------ + +.. automodule:: s3_encryption + :members: S3EncryptionClient, S3EncryptionClientConfig + +Materials +--------- + +KMS Keyring +~~~~~~~~~~~ + +.. automodule:: s3_encryption.materials.kms_keyring + :members: + +Keyring Interface +~~~~~~~~~~~~~~~~~ + +.. automodule:: s3_encryption.materials.keyring + :members: + +Materials +~~~~~~~~~ + +.. automodule:: s3_encryption.materials.materials + :members: + +Exceptions +---------- + +.. automodule:: s3_encryption.exceptions + :members: diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..584e5145 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,38 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Sphinx configuration for Amazon S3 Encryption Client for Python.""" + +project = "Amazon S3 Encryption Client for Python" +copyright = "Amazon.com, Inc. or its affiliates" +author = "AWS Crypto Tools" + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx.ext.intersphinx", + "sphinx.ext.viewcode", +] + +# Napoleon settings for Google-style docstrings +napoleon_google_docstring = True +napoleon_numpy_docstring = False + +# Autodoc settings +autodoc_member_order = "bysource" +autodoc_default_options = { + "members": True, + "undoc-members": False, + "show-inheritance": True, +} + +# Intersphinx mappings +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "boto3": ("https://boto3.amazonaws.com/v1/documentation/api/latest/", None), +} + +# Theme +html_theme = "sphinx_rtd_theme" + +# Exclude patterns +exclude_patterns = ["_build"] diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..84dd359a --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,41 @@ +Amazon S3 Encryption Client for Python +======================================= + +The Amazon S3 Encryption Client for Python provides client-side encryption +for objects stored in Amazon S3. It wraps a standard boto3 S3 client and +transparently encrypts objects on upload and decrypts them on download. + +.. toctree:: + :maxdepth: 2 + :caption: Contents + + api + +Getting Started +--------------- + +.. code-block:: python + + import boto3 + from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig + from s3_encryption.materials.kms_keyring import KmsKeyring + + kms_client = boto3.client("kms", region_name="us-west-2") + keyring = KmsKeyring(kms_client, "arn:aws:kms:us-west-2:123456789012:alias/my-key") + + s3_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring=keyring) + s3ec = S3EncryptionClient(s3_client, config) + + # Encrypt and upload + s3ec.put_object(Bucket="my-bucket", Key="my-object", Body=b"secret data") + + # Download and decrypt + response = s3ec.get_object(Bucket="my-bucket", Key="my-object") + plaintext = response["Body"].read() + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` diff --git a/pyproject.toml b/pyproject.toml index 25318942..abad14d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,10 @@ dev = [ "ruff>=0.15.12", "boto3-stubs~=1.43.6", ] +docs = [ + "sphinx>=7.0,<8", + "sphinx-rtd-theme>=2.0,<3", +] [build-system] requires = ["hatchling"] diff --git a/release-validation/validate.py b/release-validation/validate.py new file mode 100644 index 00000000..c9af1ef3 --- /dev/null +++ b/release-validation/validate.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Post-release validation: install the published package and do a round-trip. + +This script is run after publishing to TestPyPI or PyPI to verify that +the released artifact works correctly for consumers. +""" + +import os +import sys +import uuid + +import boto3 + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption._utils import _PACKAGE_VERSION +from s3_encryption.materials.kms_keyring import KmsKeyring + +BUCKET = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") +KMS_KEY_ID = os.environ.get( + "CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" +) +REGION = "us-west-2" + + +def main(): + print(f"Validating amazon-s3-encryption-client-python v{_PACKAGE_VERSION}") + + kms_client = boto3.client("kms", region_name=REGION) + keyring = KmsKeyring(kms_client, KMS_KEY_ID) + s3_client = boto3.client("s3", region_name=REGION) + config = S3EncryptionClientConfig(keyring=keyring) + s3ec = S3EncryptionClient(s3_client, config) + + key = f"release-validation/{uuid.uuid4()}" + plaintext = b"Release validation round-trip test" + + # Put + print(f" Encrypting and uploading to s3://{BUCKET}/{key}") + s3ec.put_object(Bucket=BUCKET, Key=key, Body=plaintext) + + # Get + print(f" Downloading and decrypting from s3://{BUCKET}/{key}") + response = s3ec.get_object(Bucket=BUCKET, Key=key) + result = response["Body"].read() + + assert result == plaintext, f"Round-trip failed: expected {plaintext!r}, got {result!r}" + + # Cleanup + s3_client.delete_object(Bucket=BUCKET, Key=key) + + print(" Round-trip validation passed!") + print(f" Version: {_PACKAGE_VERSION}") + print(f" User-Agent includes: S3ECPy/{_PACKAGE_VERSION}") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index e06ca9e1..ea6f1dc8 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -396,8 +396,8 @@ def put_object(self, **kwargs): The response from the S3 client's put_object method. Raises: - S3EncryptionClientError: Any problem with encryption, including if the Body parameter - has an invalid type. + S3EncryptionClientError: Any problem with encryption, including if + the Body parameter has an invalid type. """ # Extract EncryptionContext if provided (not a standard S3 parameter) encryption_context = kwargs.pop("EncryptionContext", None) From c835e6cb6d5053b53f62d33f9c54a515c0f4ddb8 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier <76071473+kessplas@users.noreply.github.com> Date: Wed, 20 May 2026 15:36:59 -0700 Subject: [PATCH 81/81] chore: special character fix for windows (#188) --- .github/workflows/python-integ.yml | 1 + .github/workflows/python-perf.yml | 1 + .github/workflows/release.yml | 2 ++ .github/workflows/test-server.yml | 1 + 4 files changed, 5 insertions(+) diff --git a/.github/workflows/python-integ.yml b/.github/workflows/python-integ.yml index 6a23a7b3..7c22d3e4 100644 --- a/.github/workflows/python-integ.yml +++ b/.github/workflows/python-integ.yml @@ -51,6 +51,7 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v6 with: + special-characters-workaround: "true" role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role aws-region: us-west-2 diff --git a/.github/workflows/python-perf.yml b/.github/workflows/python-perf.yml index 8b4bed00..38bddb56 100644 --- a/.github/workflows/python-perf.yml +++ b/.github/workflows/python-perf.yml @@ -44,6 +44,7 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v6 with: + special-characters-workaround: "true" role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role aws-region: us-west-2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 72742fe0..f43a2abc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -118,6 +118,7 @@ jobs: python-version: "3.10" - uses: aws-actions/configure-aws-credentials@v6 with: + special-characters-workaround: "true" role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role aws-region: us-west-2 - name: Wait for TestPyPI availability @@ -170,6 +171,7 @@ jobs: python-version: "3.10" - uses: aws-actions/configure-aws-credentials@v6 with: + special-characters-workaround: "true" role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role aws-region: us-west-2 - name: Wait for PyPI availability diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml index 7b26c3fc..b60c2167 100644 --- a/.github/workflows/test-server.yml +++ b/.github/workflows/test-server.yml @@ -102,6 +102,7 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v6 with: + special-characters-workaround: "true" role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role aws-region: us-west-2