diff --git a/.duvet/.gitignore b/.duvet/.gitignore new file mode 100644 index 00000000..93956e36 --- /dev/null +++ b/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ \ No newline at end of file diff --git a/.duvet/config.toml b/.duvet/config.toml new file mode 100644 index 00000000..41d29170 --- /dev/null +++ b/.duvet/config.toml @@ -0,0 +1,33 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "src/**/*.py" +type = "implementation" +comment-style = { meta = "##=", content = "##%" } +[[source]] +pattern = "test/**/*.py" +type = "test" +comment-style = { meta = "##=", content = "##%" } + +# Include required specifications here +[[specification]] +source = "specification/s3-encryption/client.md" +[[specification]] +source = "specification/s3-encryption/decryption.md" +[[specification]] +source = "specification/s3-encryption/encryption.md" +[[specification]] +source = "specification/s3-encryption/key-commitment.md" +[[specification]] +source = "specification/s3-encryption/key-derivation.md" +[[specification]] +source = "specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "specification/s3-encryption/data-format/metadata-strategy.md" + +[report.html] +enabled = true + +# Enable snapshots to prevent requirement coverage regressions +[report.snapshot] +enabled = false diff --git a/.github/workflows/all-ci.yml b/.github/workflows/all-ci.yml index c65364b7..e35342fc 100644 --- a/.github/workflows/all-ci.yml +++ b/.github/workflows/all-ci.yml @@ -37,6 +37,15 @@ jobs: python-version: ${{ inputs.python-version || '3.11' }} secrets: inherit + run-duvet: + permissions: + id-token: write + contents: read + pages: write + name: Run Duvet + uses: ./.github/workflows/duvet.yml + secrets: inherit + run-duvet-test-server: permissions: id-token: write diff --git a/.github/workflows/duvet-test-server.yml b/.github/workflows/duvet-test-server.yml index f4bac5a8..f8f6e2ac 100644 --- a/.github/workflows/duvet-test-server.yml +++ b/.github/workflows/duvet-test-server.yml @@ -15,9 +15,31 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v5 - with: - submodules: true - token: ${{ secrets.PAT_FOR_SPEC }} + + # There are a lot of submodules here + # This initializes the checkouts in parallel (--jobs) + # rather than in series the way actions/checkout@v5 does it. + + - name: Get CPU count + id: cpu-count + run: echo "count=$(node -p 'require("os").cpus().length')" >> $GITHUB_OUTPUT + + - name: Setup git submodules with PAT + run: | + git config --global url."https://github.com/".insteadOf "git@github.com:" + git config --global credential.helper store + echo "https://x-token-auth:${{ secrets.PAT_FOR_SPEC }}@github.com" > ~/.git-credentials + + - name: Optimize git for performance + run: | + git config --global fetch.parallel ${{ steps.cpu-count.outputs.count }} + git config --global submodule.fetchJobs ${{ steps.cpu-count.outputs.count }} + git config --global remote.origin.tagOpt --no-tags + + - name: Checkout submodules with --jobs + run: | + git submodule update --init --depth 1 --single-branch --jobs ${{ steps.cpu-count.outputs.count }} test-server/ + - name: Checkout CPP code cpp-v3 uses: actions/checkout@v5 @@ -32,14 +54,9 @@ jobs: with: toolchain: stable - - name: Clone duvet repository - run: git clone https://github.com/awslabs/duvet.git /tmp/duvet - - name: Build and install duvet run: | - cd /tmp/duvet - cargo xtask build - cargo install --path ./duvet + cargo install duvet --locked - name: Run duvet if: always() @@ -49,7 +66,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: reports + name: test-server-reports include-hidden-files: true path: test-server/*-server/.duvet/reports/report.html diff --git a/.github/workflows/duvet.yml b/.github/workflows/duvet.yml new file mode 100644 index 00000000..eb7b49e2 --- /dev/null +++ b/.github/workflows/duvet.yml @@ -0,0 +1,40 @@ +name: duvet on the local S3EC-Python + +on: + workflow_call: + # Optional inputs that can be provided when calling this workflow + +jobs: + test: + runs-on: ubuntu-slim + permissions: + id-token: write + contents: read + pages: write + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Checkout specific specification + run: git submodule update --init --recursive specification + + - name: Setup Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: stable + + - name: Install duvet + run: | + cargo install duvet --locked + + - name: Run duvet + run: make duvet + + - name: Upload duvet reports + uses: actions/upload-artifact@v4 + with: + name: reports + include-hidden-files: true + path: .duvet/reports/report.html + diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml index 4fa10666..80991a99 100644 --- a/.github/workflows/test-server.yml +++ b/.github/workflows/test-server.yml @@ -46,7 +46,7 @@ jobs: - name: Checkout submodules with --jobs run: | - git submodule update --init --depth 1 --single-branch --jobs ${{ steps.cpu-count.outputs.count }} + git submodule update --init --depth 1 --single-branch --jobs ${{ steps.cpu-count.outputs.count }} test-server/ - name: Update cpp submodules recursively with --jobs run: | diff --git a/.gitmodules b/.gitmodules index 75e91f99..162cd457 100644 --- a/.gitmodules +++ b/.gitmodules @@ -46,3 +46,7 @@ path = test-server/cpp-v3-server/aws-sdk-cpp url = git@github.com:aws/aws-sdk-cpp.git branch = main +[submodule "specification"] + path = specification + url = https://github.com/awslabs/aws-encryption-sdk-specification.git + branch = tonyknap/s3ec-v3.0.1-candidate diff --git a/Makefile b/Makefile index 814ab334..405ba24f 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: lint format test test-unit test-integration install # Default target -all: lint test +all: lint test duvet # Install dependencies install: @@ -37,3 +37,13 @@ clean: find . -type d -name .pytest_cache -exec rm -rf {} + find . -type d -name .coverage -exec rm -rf {} + find . -type f -name "*.pyc" -delete + rm -rf .duvet/reports/ .duvet/requirements/ + +duvet: | clean duvet-report + +duvet-report: + duvet report + +duvet-view-report-mac: + open .duvet/reports/report.html + diff --git a/specification b/specification new file mode 160000 index 00000000..7edabc2a --- /dev/null +++ b/specification @@ -0,0 +1 @@ +Subproject commit 7edabc2a69890e1119c49f948a086638182369a4 diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 37093803..2e7d264c 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -149,6 +149,13 @@ def decrypt(self, response, encryption_context=None): # Get decryption materials from the crypto materials manager dec_materials = self.cmm.decrypt_materials(dec_materials) + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=TODO + ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and + ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is NOT enabled, + ##% the S3EC MUST throw an error which details that client was + ##% not configured to decrypt objects with ALG_AES_256_CBC_IV16_NO_KDF. + aesgcm = AESGCM(dec_materials.plaintext_data_key) return aesgcm.decrypt(nonce=iv_bytes, data=encrypted_data, associated_data=None)