From 2e699d9e51bffe8735d8587f46a0d3260873981a Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Thu, 12 Feb 2026 16:02:04 -0600 Subject: [PATCH 1/7] ci(duvet): add spec and duvet for Python --- .duvet/.gitignore | 3 +++ .duvet/config.toml | 33 +++++++++++++++++++++++++ .github/workflows/all-ci.yml | 9 +++++++ .github/workflows/duvet.yml | 45 ++++++++++++++++++++++++++++++++++ .gitmodules | 4 +++ Makefile | 12 ++++++++- specification | 1 + src/s3_encryption/pipelines.py | 5 ++++ 8 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 .duvet/.gitignore create mode 100644 .duvet/config.toml create mode 100644 .github/workflows/duvet.yml create mode 160000 specification diff --git a/.duvet/.gitignore b/.duvet/.gitignore new file mode 100644 index 00000000..93956e36 --- /dev/null +++ b/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ \ No newline at end of file diff --git a/.duvet/config.toml b/.duvet/config.toml new file mode 100644 index 00000000..41d29170 --- /dev/null +++ b/.duvet/config.toml @@ -0,0 +1,33 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "src/**/*.py" +type = "implementation" +comment-style = { meta = "##=", content = "##%" } +[[source]] +pattern = "test/**/*.py" +type = "test" +comment-style = { meta = "##=", content = "##%" } + +# Include required specifications here +[[specification]] +source = "specification/s3-encryption/client.md" +[[specification]] +source = "specification/s3-encryption/decryption.md" +[[specification]] +source = "specification/s3-encryption/encryption.md" +[[specification]] +source = "specification/s3-encryption/key-commitment.md" +[[specification]] +source = "specification/s3-encryption/key-derivation.md" +[[specification]] +source = "specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "specification/s3-encryption/data-format/metadata-strategy.md" + +[report.html] +enabled = true + +# Enable snapshots to prevent requirement coverage regressions +[report.snapshot] +enabled = false diff --git a/.github/workflows/all-ci.yml b/.github/workflows/all-ci.yml index c65364b7..e35342fc 100644 --- a/.github/workflows/all-ci.yml +++ b/.github/workflows/all-ci.yml @@ -37,6 +37,15 @@ jobs: python-version: ${{ inputs.python-version || '3.11' }} secrets: inherit + run-duvet: + permissions: + id-token: write + contents: read + pages: write + name: Run Duvet + uses: ./.github/workflows/duvet.yml + secrets: inherit + run-duvet-test-server: permissions: id-token: write diff --git a/.github/workflows/duvet.yml b/.github/workflows/duvet.yml new file mode 100644 index 00000000..18f899a5 --- /dev/null +++ b/.github/workflows/duvet.yml @@ -0,0 +1,45 @@ +name: duvet on the local S3EC-Python + +on: + workflow_call: + # Optional inputs that can be provided when calling this workflow + +jobs: + test: + runs-on: ubuntu-slim + permissions: + id-token: write + contents: read + pages: write + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Checkout specific specification + run: git submodule update --init --recursive specification + + - name: Setup Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: stable + + - name: Clone duvet repository + run: git clone https://github.com/awslabs/duvet.git /tmp/duvet + + - name: Build and install duvet + run: | + cd /tmp/duvet + cargo xtask build + cargo install --path ./duvet + + - name: Run duvet + run: make duvet + + - name: Upload duvet reports + uses: actions/upload-artifact@v4 + with: + name: reports + include-hidden-files: true + path: .duvet/reports/report.html + diff --git a/.gitmodules b/.gitmodules index 75e91f99..162cd457 100644 --- a/.gitmodules +++ b/.gitmodules @@ -46,3 +46,7 @@ path = test-server/cpp-v3-server/aws-sdk-cpp url = git@github.com:aws/aws-sdk-cpp.git branch = main +[submodule "specification"] + path = specification + url = https://github.com/awslabs/aws-encryption-sdk-specification.git + branch = tonyknap/s3ec-v3.0.1-candidate diff --git a/Makefile b/Makefile index 814ab334..405ba24f 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: lint format test test-unit test-integration install # Default target -all: lint test +all: lint test duvet # Install dependencies install: @@ -37,3 +37,13 @@ clean: find . -type d -name .pytest_cache -exec rm -rf {} + find . -type d -name .coverage -exec rm -rf {} + find . -type f -name "*.pyc" -delete + rm -rf .duvet/reports/ .duvet/requirements/ + +duvet: | clean duvet-report + +duvet-report: + duvet report + +duvet-view-report-mac: + open .duvet/reports/report.html + diff --git a/specification b/specification new file mode 160000 index 00000000..2e1710a6 --- /dev/null +++ b/specification @@ -0,0 +1 @@ +Subproject commit 2e1710a6b305484612951bc97985fd15c80f5823 diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 37093803..eb363bb1 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -149,6 +149,11 @@ def decrypt(self, response, encryption_context=None): # Get decryption materials from the crypto materials manager dec_materials = self.cmm.decrypt_materials(dec_materials) + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=TODO + ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and [legacy unauthenticated algorithm suites](#legacy-decryption) is NOT enabled, + ##% the S3EC MUST throw an error which details that client was not configured to decrypt objects with ALG_AES_256_CBC_IV16_NO_KDF. + aesgcm = AESGCM(dec_materials.plaintext_data_key) return aesgcm.decrypt(nonce=iv_bytes, data=encrypted_data, associated_data=None) From 2a4e16593d947f8febd565b0f12d6ba94666b922 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Thu, 12 Feb 2026 16:14:49 -0600 Subject: [PATCH 2/7] ci(GHW): optimize submodule checkout Test Servers do not rely on the root/spec submodule, but all other submodules. --- .github/workflows/duvet-test-server.yml | 28 ++++++++++++++++++++++--- .github/workflows/test-server.yml | 2 +- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/duvet-test-server.yml b/.github/workflows/duvet-test-server.yml index f4bac5a8..1219b145 100644 --- a/.github/workflows/duvet-test-server.yml +++ b/.github/workflows/duvet-test-server.yml @@ -15,9 +15,31 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v5 - with: - submodules: true - token: ${{ secrets.PAT_FOR_SPEC }} + + # There are a lot of submodules here + # This initializes the checkouts in parallel (--jobs) + # rather than in series the way actions/checkout@v5 does it. + + - name: Get CPU count + id: cpu-count + run: echo "count=$(node -p 'require("os").cpus().length')" >> $GITHUB_OUTPUT + + - name: Setup git submodules with PAT + run: | + git config --global url."https://github.com/".insteadOf "git@github.com:" + git config --global credential.helper store + echo "https://x-token-auth:${{ secrets.PAT_FOR_SPEC }}@github.com" > ~/.git-credentials + + - name: Optimize git for performance + run: | + git config --global fetch.parallel ${{ steps.cpu-count.outputs.count }} + git config --global submodule.fetchJobs ${{ steps.cpu-count.outputs.count }} + git config --global remote.origin.tagOpt --no-tags + + - name: Checkout submodules with --jobs + run: | + git submodule update --init --depth 1 --single-branch --jobs ${{ steps.cpu-count.outputs.count }} test-server/ + - name: Checkout CPP code cpp-v3 uses: actions/checkout@v5 diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml index 4fa10666..80991a99 100644 --- a/.github/workflows/test-server.yml +++ b/.github/workflows/test-server.yml @@ -46,7 +46,7 @@ jobs: - name: Checkout submodules with --jobs run: | - git submodule update --init --depth 1 --single-branch --jobs ${{ steps.cpu-count.outputs.count }} + git submodule update --init --depth 1 --single-branch --jobs ${{ steps.cpu-count.outputs.count }} test-server/ - name: Update cpp submodules recursively with --jobs run: | From 7e98d99e63c30473813df05ed064d46b854bce50 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Thu, 12 Feb 2026 16:52:26 -0600 Subject: [PATCH 3/7] ci(duvet): conflicting names --- .github/workflows/duvet-test-server.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/duvet-test-server.yml b/.github/workflows/duvet-test-server.yml index 1219b145..c917f824 100644 --- a/.github/workflows/duvet-test-server.yml +++ b/.github/workflows/duvet-test-server.yml @@ -71,7 +71,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: reports + name: test-server-reports include-hidden-files: true path: test-server/*-server/.duvet/reports/report.html From 5220a657bf376c33208cf4592630c1dc9e7849ee Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Thu, 12 Feb 2026 16:55:18 -0600 Subject: [PATCH 4/7] ci(duvet): duvet makes for long lines --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index b05f22aa..e2c2965e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,3 +55,4 @@ known-first-party = ["s3_encryption"] [tool.ruff.lint.per-file-ignores] "test/**/*.py" = ["D100", "D101", "D102", "D103", "D104", "E501"] +"src/**/*.py" = ["E501"] \ No newline at end of file From 73a755306e3a757630e2ad854fb53362858edb18 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 16 Feb 2026 10:32:40 -0600 Subject: [PATCH 5/7] chore(duvet): keep line spacing --- pyproject.toml | 1 - src/s3_encryption/pipelines.py | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e2c2965e..b05f22aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,4 +55,3 @@ known-first-party = ["s3_encryption"] [tool.ruff.lint.per-file-ignores] "test/**/*.py" = ["D100", "D101", "D102", "D103", "D104", "E501"] -"src/**/*.py" = ["E501"] \ No newline at end of file diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index eb363bb1..2e7d264c 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -151,8 +151,10 @@ def decrypt(self, response, encryption_context=None): ##= specification/s3-encryption/decryption.md#cbc-decryption ##= type=TODO - ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and [legacy unauthenticated algorithm suites](#legacy-decryption) is NOT enabled, - ##% the S3EC MUST throw an error which details that client was not configured to decrypt objects with ALG_AES_256_CBC_IV16_NO_KDF. + ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and + ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is NOT enabled, + ##% the S3EC MUST throw an error which details that client was + ##% not configured to decrypt objects with ALG_AES_256_CBC_IV16_NO_KDF. aesgcm = AESGCM(dec_materials.plaintext_data_key) From 0bef3a3540788403e2b6377ee8bd742e278605d8 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Tue, 17 Feb 2026 14:45:24 -0600 Subject: [PATCH 6/7] chore(duvet): bump spec to head --- specification | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification b/specification index 2e1710a6..7edabc2a 160000 --- a/specification +++ b/specification @@ -1 +1 @@ -Subproject commit 2e1710a6b305484612951bc97985fd15c80f5823 +Subproject commit 7edabc2a69890e1119c49f948a086638182369a4 From c09ab009fa687e7c69adb4aee7ae7b87437b520e Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Tue, 17 Feb 2026 14:54:03 -0600 Subject: [PATCH 7/7] ci(duvet): just install from cargo --- .github/workflows/duvet-test-server.yml | 7 +------ .github/workflows/duvet.yml | 9 ++------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/.github/workflows/duvet-test-server.yml b/.github/workflows/duvet-test-server.yml index c917f824..f8f6e2ac 100644 --- a/.github/workflows/duvet-test-server.yml +++ b/.github/workflows/duvet-test-server.yml @@ -54,14 +54,9 @@ jobs: with: toolchain: stable - - name: Clone duvet repository - run: git clone https://github.com/awslabs/duvet.git /tmp/duvet - - name: Build and install duvet run: | - cd /tmp/duvet - cargo xtask build - cargo install --path ./duvet + cargo install duvet --locked - name: Run duvet if: always() diff --git a/.github/workflows/duvet.yml b/.github/workflows/duvet.yml index 18f899a5..eb7b49e2 100644 --- a/.github/workflows/duvet.yml +++ b/.github/workflows/duvet.yml @@ -24,14 +24,9 @@ jobs: with: toolchain: stable - - name: Clone duvet repository - run: git clone https://github.com/awslabs/duvet.git /tmp/duvet - - - name: Build and install duvet + - name: Install duvet run: | - cd /tmp/duvet - cargo xtask build - cargo install --path ./duvet + cargo install duvet --locked - name: Run duvet run: make duvet