From ce13712aecb0903dfa688001c104619ec2ed9a1c Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Tue, 7 Apr 2026 17:46:30 -0700 Subject: [PATCH 1/9] chore: add perf suite --- .github/workflows/all-ci.yml | 10 + .github/workflows/python-perf.yml | 66 ++++++ .gitignore | 1 + Makefile | 8 +- test/performance/__init__.py | 2 + test/performance/conftest.py | 60 +++++ test/performance/generate_report.py | 120 ++++++++++ test/performance/test_perf_s3_encryption.py | 247 ++++++++++++++++++++ 8 files changed, 512 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/python-perf.yml create mode 100644 test/performance/__init__.py create mode 100644 test/performance/conftest.py create mode 100644 test/performance/generate_report.py create mode 100644 test/performance/test_perf_s3_encryption.py diff --git a/.github/workflows/all-ci.yml b/.github/workflows/all-ci.yml index e35342fc..d8c92bb9 100644 --- a/.github/workflows/all-ci.yml +++ b/.github/workflows/all-ci.yml @@ -37,6 +37,16 @@ jobs: python-version: ${{ inputs.python-version || '3.11' }} secrets: inherit + python-perf: + permissions: + id-token: write + contents: read + name: Python Performance Tests + uses: ./.github/workflows/python-perf.yml + with: + python-version: ${{ inputs.python-version || '3.11' }} + secrets: inherit + run-duvet: permissions: id-token: write diff --git a/.github/workflows/python-perf.yml b/.github/workflows/python-perf.yml new file mode 100644 index 00000000..2e6bee40 --- /dev/null +++ b/.github/workflows/python-perf.yml @@ -0,0 +1,66 @@ +name: Python Performance Tests + +on: + workflow_call: + inputs: + python-version: + description: "Python version to use" + default: "3.11" + required: false + type: string + +jobs: + python-perf: + runs-on: macos-14-large + permissions: + id-token: write + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + submodules: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ inputs.python-version || '3.11' }} + + - name: Cache uv dependencies + uses: actions/cache@v5 + with: + path: ~/.cache/uv + key: ${{ runner.os }}-uv-${{ hashFiles('**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-uv- + + - name: Install Uv + run: pip install uv + + - name: Install dependencies + run: make install + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role + aws-region: us-west-2 + + - name: Run performance tests + run: make test-perf + env: + CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} + CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} + PERF_NUM_ROUNDS: "5" + + - name: Generate performance HTML report + if: always() + run: uv run python test/performance/generate_report.py + + - name: Upload performance report + if: always() + uses: actions/upload-artifact@v7 + with: + name: performance-report + path: perf-results/ diff --git a/.gitignore b/.gitignore index 3691eef4..b0b67407 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,4 @@ smithy-java-core/out *.pid .coverage coverage-report/ +perf-results/ diff --git a/Makefile b/Makefile index 256f50b7..d8e9966c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: lint format test test-unit test-integration install +.PHONY: lint format test test-unit test-integration test-perf install # Default target all: lint test duvet @@ -25,12 +25,16 @@ test: test-unit test-integration # Run unit tests (creates .coverage report) test-unit: - uv run pytest test/ --ignore=test/integration/ --verbose --cov=src/s3_encryption --cov-report=term-missing + uv run pytest test/ --ignore=test/integration/ --ignore=test/performance/ --verbose --cov=src/s3_encryption --cov-report=term-missing # Run integration tests (appends to .coverage report from test-unit) test-integration: uv run pytest test/integration/ --verbose --cov=src/s3_encryption --cov-append --cov-report=term-missing +# Run performance tests +test-perf: + uv run pytest test/performance/ --verbose -x + # Clean up cache files clean: find . -type d -name __pycache__ -exec rm -rf {} + diff --git a/test/performance/__init__.py b/test/performance/__init__.py new file mode 100644 index 00000000..f94fd12a --- /dev/null +++ b/test/performance/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/test/performance/conftest.py b/test/performance/conftest.py new file mode 100644 index 00000000..e9843ff2 --- /dev/null +++ b/test/performance/conftest.py @@ -0,0 +1,60 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Shared fixtures for performance tests.""" + +import os + +import boto3 +import pytest + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy + +BUCKET = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") +REGION = os.environ.get("CI_AWS_REGION", "us-west-2") +KMS_KEY_ID = os.environ.get( + "CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" +) + +# Performance test configuration +NUM_ROUNDS = int(os.environ.get("PERF_NUM_ROUNDS", "5")) +OBJECT_SIZES_MB = [10, 25, 50] + + +def _make_s3ec(algorithm_suite, commitment_policy): + kms_client = boto3.client("kms", region_name=REGION) + keyring = KmsKeyring(kms_client, KMS_KEY_ID) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig( + keyring, + encryption_algorithm=algorithm_suite, + commitment_policy=commitment_policy, + ) + return S3EncryptionClient(wrapped_client, config) + + +@pytest.fixture(scope="module") +def s3ec_v2(): + return _make_s3ec( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) + + +@pytest.fixture(scope="module") +def s3ec_v3(): + return _make_s3ec( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + + +@pytest.fixture(scope="module") +def plain_s3(): + return boto3.client("s3", region_name=REGION) + + +@pytest.fixture(scope="module") +def kms_client(): + return boto3.client("kms", region_name=REGION) diff --git a/test/performance/generate_report.py b/test/performance/generate_report.py new file mode 100644 index 00000000..ade7f746 --- /dev/null +++ b/test/performance/generate_report.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Generate an HTML performance report from the JSON results file.""" + +import json +import sys +from pathlib import Path + +RESULTS_FILE = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("perf-results/results.json") +OUTPUT_FILE = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("perf-results/report.html") + + +def _fmt(seconds: float) -> str: + if seconds < 1: + return f"{seconds * 1000:.1f} ms" + return f"{seconds:.2f} s" + + +def generate_html(data: dict) -> str: + config = data["config"] + results = data["results"] + timestamp = data["timestamp"] + + # Group results by category + groups: dict[str, list[dict]] = {} + for r in results: + name = r["test"] + if "roundtrip" in name or "local_crypto" in name: + cat = "Roundtrip: S3EC vs Local Crypto + Plain S3" + elif "put" in name and "plain" in name: + cat = "Put: S3EC vs Plain S3" + elif "get" in name and "plain" in name: + cat = "Get: S3EC vs Plain S3" + elif "put" in name: + cat = "Put by Algorithm Suite" + elif "get" in name: + cat = "Get by Algorithm Suite" + else: + cat = "Other" + groups.setdefault(cat, []).append(r) + + sections_html = "" + for cat, items in groups.items(): + rows = "" + for r in sorted(items, key=lambda x: (x["size_mb"], x["test"])): + durations_str = ", ".join(_fmt(d) for d in r["durations_s"]) + rows += f""" + + {r['test']} + {r['size_mb']} MB + {r['rounds']} + {_fmt(r['mean_s'])} + {_fmt(r['min_s'])} + {_fmt(r['max_s'])} + {durations_str} + """ + + sections_html += f""" +

{cat}

+ + + + + + + + {rows} + +
TestSizeRoundsMeanMinMaxAll Durations
""" + + return f""" + + + +S3EC Performance Report + + + +

S3 Encryption Client — Performance Report

+
+ Generated: {timestamp}
+ Rounds per test: {config['num_rounds']} · + Object sizes: {', '.join(str(s) + ' MB' for s in config['object_sizes_mb'])} · + Bucket: {config['bucket']} · Region: {config['region']} +
+{sections_html} + +""" + + +def main(): + if not RESULTS_FILE.exists(): + print(f"Results file not found: {RESULTS_FILE}", file=sys.stderr) + sys.exit(1) + + with open(RESULTS_FILE) as f: + data = json.load(f) + + html = generate_html(data) + OUTPUT_FILE.parent.mkdir(parents=True, exist_ok=True) + OUTPUT_FILE.write_text(html) + print(f"Report written to {OUTPUT_FILE}") + + +if __name__ == "__main__": + main() diff --git a/test/performance/test_perf_s3_encryption.py b/test/performance/test_perf_s3_encryption.py new file mode 100644 index 00000000..1b8d6d52 --- /dev/null +++ b/test/performance/test_perf_s3_encryption.py @@ -0,0 +1,247 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Performance tests comparing S3EC against plaintext S3 and local encryption + S3 upload. + +Each test runs multiple rounds with large objects to get a meaningful signal. +Results are collected via a module-scoped list and written to a JSON file +that the HTML report generator consumes. +""" + +import json +import os +import time +from datetime import datetime + +import pytest +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy + +from .conftest import BUCKET, KMS_KEY_ID, NUM_ROUNDS, OBJECT_SIZES_MB, REGION, _make_s3ec + +PERF_KEY_PREFIX = "perf-test/" +RESULTS_FILE = os.environ.get("PERF_RESULTS_FILE", "perf-results/results.json") + +# Collect all benchmark results here +_results: list[dict] = [] + + +def _generate_payload(size_mb: int) -> bytes: + """Generate a deterministic payload of the given size in MB.""" + chunk = os.urandom(1024) # 1 KB random chunk + return (chunk * 1024 * size_mb)[:size_mb * 1024 * 1024] + + +def _unique_key(prefix: str) -> str: + return PERF_KEY_PREFIX + prefix + datetime.now().strftime("%Y%m%d-%H%M%S-%f") + + +def _record(test_name, size_mb, durations): + _results.append({ + "test": test_name, + "size_mb": size_mb, + "rounds": len(durations), + "durations_s": durations, + "mean_s": sum(durations) / len(durations), + "min_s": min(durations), + "max_s": max(durations), + }) + + +# --------------------------------------------------------------------------- +# S3EC put_object vs plain S3 put_object +# --------------------------------------------------------------------------- + + +ALGORITHM_CONFIGS = [ + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + id="AES_GCM", + ), + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + id="KC_GCM", + ), +] + + +@pytest.mark.parametrize("size_mb", OBJECT_SIZES_MB) +def test_s3ec_put_vs_plain_put(plain_s3, size_mb): + """Compare S3EC put_object latency against plain S3 put_object.""" + payload = _generate_payload(size_mb) + + # Benchmark plain S3 put + plain_durations = [] + for _ in range(NUM_ROUNDS): + key = _unique_key(f"plain-put-{size_mb}mb-") + t0 = time.perf_counter() + plain_s3.put_object(Bucket=BUCKET, Key=key, Body=payload) + plain_durations.append(time.perf_counter() - t0) + _record(f"plain_s3_put_{size_mb}mb", size_mb, plain_durations) + + # Benchmark S3EC put (V3 / KC_GCM — the default) + s3ec = _make_s3ec( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + s3ec_durations = [] + for _ in range(NUM_ROUNDS): + key = _unique_key(f"s3ec-put-{size_mb}mb-") + t0 = time.perf_counter() + s3ec.put_object(Bucket=BUCKET, Key=key, Body=payload) + s3ec_durations.append(time.perf_counter() - t0) + _record(f"s3ec_put_kc_gcm_{size_mb}mb", size_mb, s3ec_durations) + + +@pytest.mark.parametrize("size_mb", OBJECT_SIZES_MB) +def test_s3ec_get_vs_plain_get(plain_s3, size_mb): + """Compare S3EC get_object latency against plain S3 get_object.""" + payload = _generate_payload(size_mb) + + # Upload a plain object and an encrypted object first + plain_key = _unique_key(f"plain-get-src-{size_mb}mb-") + plain_s3.put_object(Bucket=BUCKET, Key=plain_key, Body=payload) + + s3ec = _make_s3ec( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + enc_key = _unique_key(f"s3ec-get-src-{size_mb}mb-") + s3ec.put_object(Bucket=BUCKET, Key=enc_key, Body=payload) + + # Benchmark plain S3 get + plain_durations = [] + for _ in range(NUM_ROUNDS): + t0 = time.perf_counter() + resp = plain_s3.get_object(Bucket=BUCKET, Key=plain_key) + resp["Body"].read() + plain_durations.append(time.perf_counter() - t0) + _record(f"plain_s3_get_{size_mb}mb", size_mb, plain_durations) + + # Benchmark S3EC get + s3ec_durations = [] + for _ in range(NUM_ROUNDS): + t0 = time.perf_counter() + resp = s3ec.get_object(Bucket=BUCKET, Key=enc_key) + resp["Body"].read() + s3ec_durations.append(time.perf_counter() - t0) + _record(f"s3ec_get_kc_gcm_{size_mb}mb", size_mb, s3ec_durations) + + +# --------------------------------------------------------------------------- +# S3EC roundtrip vs local encrypt + plain S3 upload +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("size_mb", OBJECT_SIZES_MB) +def test_s3ec_roundtrip_vs_local_crypto(plain_s3, kms_client, size_mb): + """Compare S3EC roundtrip against manual local AES-GCM encrypt + plain S3 upload + download + decrypt.""" + payload = _generate_payload(size_mb) + + # --- S3EC roundtrip --- + s3ec = _make_s3ec( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + s3ec_durations = [] + for _ in range(NUM_ROUNDS): + key = _unique_key(f"s3ec-rt-{size_mb}mb-") + t0 = time.perf_counter() + s3ec.put_object(Bucket=BUCKET, Key=key, Body=payload) + resp = s3ec.get_object(Bucket=BUCKET, Key=key) + resp["Body"].read() + s3ec_durations.append(time.perf_counter() - t0) + _record(f"s3ec_roundtrip_kc_gcm_{size_mb}mb", size_mb, s3ec_durations) + + # --- Local crypto + plain S3 roundtrip --- + local_durations = [] + for _ in range(NUM_ROUNDS): + key = _unique_key(f"local-rt-{size_mb}mb-") + t0 = time.perf_counter() + + # Generate a data key via KMS (to keep key management comparable) + dk_resp = kms_client.generate_data_key(KeyId=KMS_KEY_ID, KeySpec="AES_256") + data_key = dk_resp["Plaintext"] + + # Local AES-GCM encrypt + aesgcm = AESGCM(data_key) + nonce = os.urandom(12) + ciphertext = aesgcm.encrypt(nonce, payload, None) + + # Upload ciphertext to S3 + plain_s3.put_object(Bucket=BUCKET, Key=key, Body=nonce + ciphertext) + + # Download and decrypt + resp = plain_s3.get_object(Bucket=BUCKET, Key=key) + blob = resp["Body"].read() + aesgcm.decrypt(blob[:12], blob[12:], None) + + local_durations.append(time.perf_counter() - t0) + _record(f"local_crypto_roundtrip_{size_mb}mb", size_mb, local_durations) + + +# --------------------------------------------------------------------------- +# Per-algorithm-suite comparison +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +@pytest.mark.parametrize("size_mb", OBJECT_SIZES_MB) +def test_s3ec_put_by_algorithm(size_mb, algorithm_suite, commitment_policy): + """Benchmark S3EC put_object for each algorithm suite.""" + payload = _generate_payload(size_mb) + suite_label = algorithm_suite.name.lower() + s3ec = _make_s3ec(algorithm_suite, commitment_policy) + + durations = [] + for _ in range(NUM_ROUNDS): + key = _unique_key(f"alg-put-{suite_label}-{size_mb}mb-") + t0 = time.perf_counter() + s3ec.put_object(Bucket=BUCKET, Key=key, Body=payload) + durations.append(time.perf_counter() - t0) + _record(f"s3ec_put_{suite_label}_{size_mb}mb", size_mb, durations) + + +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +@pytest.mark.parametrize("size_mb", OBJECT_SIZES_MB) +def test_s3ec_get_by_algorithm(size_mb, algorithm_suite, commitment_policy): + """Benchmark S3EC get_object for each algorithm suite.""" + payload = _generate_payload(size_mb) + suite_label = algorithm_suite.name.lower() + s3ec = _make_s3ec(algorithm_suite, commitment_policy) + + # Upload once + src_key = _unique_key(f"alg-get-src-{suite_label}-{size_mb}mb-") + s3ec.put_object(Bucket=BUCKET, Key=src_key, Body=payload) + + durations = [] + for _ in range(NUM_ROUNDS): + t0 = time.perf_counter() + resp = s3ec.get_object(Bucket=BUCKET, Key=src_key) + resp["Body"].read() + durations.append(time.perf_counter() - t0) + _record(f"s3ec_get_{suite_label}_{size_mb}mb", size_mb, durations) + + +# --------------------------------------------------------------------------- +# Write results to JSON at end of module +# --------------------------------------------------------------------------- + + +def test_zz_write_results(): + """Final test that writes collected results to a JSON file for the HTML report.""" + os.makedirs(os.path.dirname(RESULTS_FILE) or ".", exist_ok=True) + with open(RESULTS_FILE, "w") as f: + json.dump({ + "timestamp": datetime.now().isoformat(), + "config": { + "num_rounds": NUM_ROUNDS, + "object_sizes_mb": OBJECT_SIZES_MB, + "bucket": BUCKET, + "region": REGION, + }, + "results": _results, + }, f, indent=2) + print(f"\nPerformance results written to {RESULTS_FILE}") From 79a53f2411b7a245917d838041b409f6cc428b02 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Tue, 7 Apr 2026 17:56:14 -0700 Subject: [PATCH 2/9] snazzier report --- test/performance/generate_report.py | 187 +++++++++++++- test/performance/test_perf_s3_encryption.py | 261 +++++++++----------- 2 files changed, 290 insertions(+), 158 deletions(-) diff --git a/test/performance/generate_report.py b/test/performance/generate_report.py index ade7f746..26858483 100644 --- a/test/performance/generate_report.py +++ b/test/performance/generate_report.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Generate an HTML performance report from the JSON results file.""" +"""Generate an HTML performance report with tables and SVG bar charts.""" import json import sys @@ -10,6 +10,14 @@ RESULTS_FILE = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("perf-results/results.json") OUTPUT_FILE = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("perf-results/report.html") +# Chart palette +COLORS = { + "plain": "#36a2eb", + "aes_gcm": "#ff6384", + "kc_gcm": "#ff9f40", + "local": "#4bc0c0", +} + def _fmt(seconds: float) -> str: if seconds < 1: @@ -17,25 +25,158 @@ def _fmt(seconds: float) -> str: return f"{seconds:.2f} s" -def generate_html(data: dict) -> str: - config = data["config"] - results = data["results"] - timestamp = data["timestamp"] +def _lookup(results, prefix, size_mb): + """Find a result entry matching prefix and size.""" + for r in results: + if r["test"].startswith(prefix) and r["size_mb"] == size_mb: + return r + return None + + +def _bar_chart_svg(chart_id, title, groups, sizes, width=700, bar_h=28, gap=6): + """Render a grouped horizontal bar chart as an SVG string. + + Args: + chart_id: unique id for the SVG element + title: chart title + groups: list of dicts {label, color, values: {size_mb: mean_s}} + sizes: list of size_mb values + width: total SVG width + bar_h: height of each bar + gap: vertical gap between bars + """ + label_col_w = 120 # left column for size labels + chart_w = width - label_col_w - 80 # room for value labels on right + n_groups = len(groups) + block_h = n_groups * (bar_h + gap) + 20 # per size block + total_h = len(sizes) * block_h + 60 # extra for title + legend + + # Find max value for scaling + max_val = 0 + for g in groups: + for v in g["values"].values(): + max_val = max(max_val, v) + if max_val == 0: + max_val = 1 + + svg_parts = [ + f'', + f'{title}', + ] + + # Legend + lx = label_col_w + for g in groups: + svg_parts.append( + f'' + ) + svg_parts.append( + f'{g["label"]}' + ) + lx += len(g["label"]) * 7 + 30 + + y_offset = 58 + for size in sizes: + # Size label + svg_parts.append( + f'{size} MB' + ) + for i, g in enumerate(groups): + val = g["values"].get(size, 0) + bar_w = max(2, (val / max_val) * chart_w) + by = y_offset + i * (bar_h + gap) + svg_parts.append( + f'' + ) + svg_parts.append( + f'{_fmt(val)}' + ) + y_offset += block_h + + svg_parts.append("") + return "\n".join(svg_parts) + + +def _build_charts(results, sizes): + """Build all SVG charts from the results data.""" + charts = [] + # --- Chart 1: Put — Plain S3 vs S3EC (AES_GCM) vs S3EC (KC_GCM) --- + put_groups = [ + {"label": "Plain S3", "color": COLORS["plain"], "values": {}}, + {"label": "S3EC AES_GCM", "color": COLORS["aes_gcm"], "values": {}}, + {"label": "S3EC KC_GCM", "color": COLORS["kc_gcm"], "values": {}}, + ] + for s in sizes: + r = _lookup(results, f"plain_s3_put_{s}mb", s) + if r: + put_groups[0]["values"][s] = r["mean_s"] + r = _lookup(results, f"s3ec_put_aes_gcm_{s}mb", s) + if r: + put_groups[1]["values"][s] = r["mean_s"] + r = _lookup(results, f"s3ec_put_kc_gcm_{s}mb", s) + if r: + put_groups[2]["values"][s] = r["mean_s"] + charts.append(_bar_chart_svg("chart-put", "PutObject: Plain S3 vs S3EC", put_groups, sizes)) + + # --- Chart 2: Get — Plain S3 vs S3EC (AES_GCM) vs S3EC (KC_GCM) --- + get_groups = [ + {"label": "Plain S3", "color": COLORS["plain"], "values": {}}, + {"label": "S3EC AES_GCM", "color": COLORS["aes_gcm"], "values": {}}, + {"label": "S3EC KC_GCM", "color": COLORS["kc_gcm"], "values": {}}, + ] + for s in sizes: + r = _lookup(results, f"plain_s3_get_{s}mb", s) + if r: + get_groups[0]["values"][s] = r["mean_s"] + r = _lookup(results, f"s3ec_get_aes_gcm_{s}mb", s) + if r: + get_groups[1]["values"][s] = r["mean_s"] + r = _lookup(results, f"s3ec_get_kc_gcm_{s}mb", s) + if r: + get_groups[2]["values"][s] = r["mean_s"] + charts.append(_bar_chart_svg("chart-get", "GetObject: Plain S3 vs S3EC", get_groups, sizes)) + + # --- Chart 3: Roundtrip — S3EC (AES_GCM) vs S3EC (KC_GCM) vs Local Crypto + Plain S3 --- + rt_groups = [ + {"label": "Local Crypto + Plain S3", "color": COLORS["local"], "values": {}}, + {"label": "S3EC AES_GCM", "color": COLORS["aes_gcm"], "values": {}}, + {"label": "S3EC KC_GCM", "color": COLORS["kc_gcm"], "values": {}}, + ] + for s in sizes: + r = _lookup(results, f"local_crypto_roundtrip_{s}mb", s) + if r: + rt_groups[0]["values"][s] = r["mean_s"] + r = _lookup(results, f"s3ec_roundtrip_aes_gcm_{s}mb", s) + if r: + rt_groups[1]["values"][s] = r["mean_s"] + r = _lookup(results, f"s3ec_roundtrip_kc_gcm_{s}mb", s) + if r: + rt_groups[2]["values"][s] = r["mean_s"] + charts.append( + _bar_chart_svg("chart-rt", "Roundtrip: S3EC vs Local Crypto + Plain S3", rt_groups, sizes) + ) + + return "\n".join(charts) + + +def _build_table(results): + """Build the full results table HTML.""" # Group results by category groups: dict[str, list[dict]] = {} for r in results: name = r["test"] if "roundtrip" in name or "local_crypto" in name: cat = "Roundtrip: S3EC vs Local Crypto + Plain S3" - elif "put" in name and "plain" in name: - cat = "Put: S3EC vs Plain S3" - elif "get" in name and "plain" in name: - cat = "Get: S3EC vs Plain S3" elif "put" in name: - cat = "Put by Algorithm Suite" + cat = "PutObject: Plain S3 vs S3EC" elif "get" in name: - cat = "Get by Algorithm Suite" + cat = "GetObject: Plain S3 vs S3EC" else: cat = "Other" groups.setdefault(cat, []).append(r) @@ -68,6 +209,17 @@ def generate_html(data: dict) -> str: {rows} """ + return sections_html + + +def generate_html(data: dict) -> str: + config = data["config"] + results = data["results"] + timestamp = data["timestamp"] + sizes = config["object_sizes_mb"] + + charts_html = _build_charts(results, sizes) + tables_html = _build_table(results) return f""" @@ -80,6 +232,9 @@ def generate_html(data: dict) -> str: h1 {{ color: #232f3e; border-bottom: 3px solid #ff9900; padding-bottom: 0.5rem; }} h2 {{ color: #232f3e; margin-top: 2rem; }} .meta {{ color: #555; font-size: 0.9rem; margin-bottom: 1.5rem; }} + .charts {{ display: flex; flex-wrap: wrap; gap: 2rem; margin-bottom: 2rem; }} + .charts svg {{ background: #fff; border-radius: 6px; + box-shadow: 0 1px 3px rgba(0,0,0,0.1); padding: 0.5rem; }} table {{ border-collapse: collapse; width: 100%; margin-bottom: 1.5rem; background: #fff; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }} th {{ background: #232f3e; color: #fff; padding: 0.6rem 0.8rem; @@ -94,10 +249,16 @@ def generate_html(data: dict) -> str:
Generated: {timestamp}
Rounds per test: {config['num_rounds']} · - Object sizes: {', '.join(str(s) + ' MB' for s in config['object_sizes_mb'])} · + Object sizes: {', '.join(str(s) + ' MB' for s in sizes)} · Bucket: {config['bucket']} · Region: {config['region']}
-{sections_html} + +

Charts

+
+{charts_html} +
+ +{tables_html} """ diff --git a/test/performance/test_perf_s3_encryption.py b/test/performance/test_perf_s3_encryption.py index 1b8d6d52..b189eca6 100644 --- a/test/performance/test_perf_s3_encryption.py +++ b/test/performance/test_perf_s3_encryption.py @@ -25,11 +25,24 @@ # Collect all benchmark results here _results: list[dict] = [] +ALGORITHM_CONFIGS = [ + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + id="AES_GCM", + ), + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + id="KC_GCM", + ), +] + def _generate_payload(size_mb: int) -> bytes: - """Generate a deterministic payload of the given size in MB.""" + """Generate a random payload of the given size in MB.""" chunk = os.urandom(1024) # 1 KB random chunk - return (chunk * 1024 * size_mb)[:size_mb * 1024 * 1024] + return (chunk * 1024 * size_mb)[: size_mb * 1024 * 1024] def _unique_key(prefix: str) -> str: @@ -37,192 +50,146 @@ def _unique_key(prefix: str) -> str: def _record(test_name, size_mb, durations): - _results.append({ - "test": test_name, - "size_mb": size_mb, - "rounds": len(durations), - "durations_s": durations, - "mean_s": sum(durations) / len(durations), - "min_s": min(durations), - "max_s": max(durations), - }) + _results.append( + { + "test": test_name, + "size_mb": size_mb, + "rounds": len(durations), + "durations_s": durations, + "mean_s": sum(durations) / len(durations), + "min_s": min(durations), + "max_s": max(durations), + } + ) -# --------------------------------------------------------------------------- -# S3EC put_object vs plain S3 put_object -# --------------------------------------------------------------------------- +def _algo_label(algorithm_suite): + """Short human-readable label for an algorithm suite.""" + if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: + return "aes_gcm" + return "kc_gcm" -ALGORITHM_CONFIGS = [ - pytest.param( - AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, - CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, - id="AES_GCM", - ), - pytest.param( - AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, - id="KC_GCM", - ), -] +# --------------------------------------------------------------------------- +# S3EC put_object vs plain S3 put_object (per algorithm suite) +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) @pytest.mark.parametrize("size_mb", OBJECT_SIZES_MB) -def test_s3ec_put_vs_plain_put(plain_s3, size_mb): +def test_s3ec_put_vs_plain_put(plain_s3, size_mb, algorithm_suite, commitment_policy): """Compare S3EC put_object latency against plain S3 put_object.""" + label = _algo_label(algorithm_suite) payload = _generate_payload(size_mb) - # Benchmark plain S3 put - plain_durations = [] - for _ in range(NUM_ROUNDS): - key = _unique_key(f"plain-put-{size_mb}mb-") - t0 = time.perf_counter() - plain_s3.put_object(Bucket=BUCKET, Key=key, Body=payload) - plain_durations.append(time.perf_counter() - t0) - _record(f"plain_s3_put_{size_mb}mb", size_mb, plain_durations) - - # Benchmark S3EC put (V3 / KC_GCM — the default) - s3ec = _make_s3ec( - AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, - ) + # Benchmark plain S3 put (only record once per size, skip for second algo) + if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: + plain_durations = [] + for _ in range(NUM_ROUNDS): + key = _unique_key(f"plain-put-{size_mb}mb-") + t0 = time.perf_counter() + plain_s3.put_object(Bucket=BUCKET, Key=key, Body=payload) + plain_durations.append(time.perf_counter() - t0) + _record(f"plain_s3_put_{size_mb}mb", size_mb, plain_durations) + + # Benchmark S3EC put + s3ec = _make_s3ec(algorithm_suite, commitment_policy) s3ec_durations = [] for _ in range(NUM_ROUNDS): - key = _unique_key(f"s3ec-put-{size_mb}mb-") + key = _unique_key(f"s3ec-put-{label}-{size_mb}mb-") t0 = time.perf_counter() s3ec.put_object(Bucket=BUCKET, Key=key, Body=payload) s3ec_durations.append(time.perf_counter() - t0) - _record(f"s3ec_put_kc_gcm_{size_mb}mb", size_mb, s3ec_durations) + _record(f"s3ec_put_{label}_{size_mb}mb", size_mb, s3ec_durations) + + +# --------------------------------------------------------------------------- +# S3EC get_object vs plain S3 get_object (per algorithm suite) +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) @pytest.mark.parametrize("size_mb", OBJECT_SIZES_MB) -def test_s3ec_get_vs_plain_get(plain_s3, size_mb): +def test_s3ec_get_vs_plain_get(plain_s3, size_mb, algorithm_suite, commitment_policy): """Compare S3EC get_object latency against plain S3 get_object.""" + label = _algo_label(algorithm_suite) payload = _generate_payload(size_mb) - # Upload a plain object and an encrypted object first - plain_key = _unique_key(f"plain-get-src-{size_mb}mb-") - plain_s3.put_object(Bucket=BUCKET, Key=plain_key, Body=payload) + # Upload a plain object (only once per size) + if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: + plain_key = _unique_key(f"plain-get-src-{size_mb}mb-") + plain_s3.put_object(Bucket=BUCKET, Key=plain_key, Body=payload) - s3ec = _make_s3ec( - AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, - ) - enc_key = _unique_key(f"s3ec-get-src-{size_mb}mb-") - s3ec.put_object(Bucket=BUCKET, Key=enc_key, Body=payload) + plain_durations = [] + for _ in range(NUM_ROUNDS): + t0 = time.perf_counter() + resp = plain_s3.get_object(Bucket=BUCKET, Key=plain_key) + resp["Body"].read() + plain_durations.append(time.perf_counter() - t0) + _record(f"plain_s3_get_{size_mb}mb", size_mb, plain_durations) - # Benchmark plain S3 get - plain_durations = [] - for _ in range(NUM_ROUNDS): - t0 = time.perf_counter() - resp = plain_s3.get_object(Bucket=BUCKET, Key=plain_key) - resp["Body"].read() - plain_durations.append(time.perf_counter() - t0) - _record(f"plain_s3_get_{size_mb}mb", size_mb, plain_durations) + # Upload an encrypted object and benchmark get + s3ec = _make_s3ec(algorithm_suite, commitment_policy) + enc_key = _unique_key(f"s3ec-get-src-{label}-{size_mb}mb-") + s3ec.put_object(Bucket=BUCKET, Key=enc_key, Body=payload) - # Benchmark S3EC get s3ec_durations = [] for _ in range(NUM_ROUNDS): t0 = time.perf_counter() resp = s3ec.get_object(Bucket=BUCKET, Key=enc_key) resp["Body"].read() s3ec_durations.append(time.perf_counter() - t0) - _record(f"s3ec_get_kc_gcm_{size_mb}mb", size_mb, s3ec_durations) + _record(f"s3ec_get_{label}_{size_mb}mb", size_mb, s3ec_durations) # --------------------------------------------------------------------------- -# S3EC roundtrip vs local encrypt + plain S3 upload +# S3EC roundtrip vs local encrypt + plain S3 upload (per algorithm suite) # --------------------------------------------------------------------------- +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) @pytest.mark.parametrize("size_mb", OBJECT_SIZES_MB) -def test_s3ec_roundtrip_vs_local_crypto(plain_s3, kms_client, size_mb): - """Compare S3EC roundtrip against manual local AES-GCM encrypt + plain S3 upload + download + decrypt.""" +def test_s3ec_roundtrip_vs_local_crypto( + plain_s3, kms_client, size_mb, algorithm_suite, commitment_policy +): + """Compare S3EC roundtrip against manual local AES-GCM encrypt + plain S3 roundtrip.""" + label = _algo_label(algorithm_suite) payload = _generate_payload(size_mb) - # --- S3EC roundtrip --- - s3ec = _make_s3ec( - AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, - ) + # S3EC roundtrip + s3ec = _make_s3ec(algorithm_suite, commitment_policy) s3ec_durations = [] for _ in range(NUM_ROUNDS): - key = _unique_key(f"s3ec-rt-{size_mb}mb-") + key = _unique_key(f"s3ec-rt-{label}-{size_mb}mb-") t0 = time.perf_counter() s3ec.put_object(Bucket=BUCKET, Key=key, Body=payload) resp = s3ec.get_object(Bucket=BUCKET, Key=key) resp["Body"].read() s3ec_durations.append(time.perf_counter() - t0) - _record(f"s3ec_roundtrip_kc_gcm_{size_mb}mb", size_mb, s3ec_durations) - - # --- Local crypto + plain S3 roundtrip --- - local_durations = [] - for _ in range(NUM_ROUNDS): - key = _unique_key(f"local-rt-{size_mb}mb-") - t0 = time.perf_counter() - - # Generate a data key via KMS (to keep key management comparable) - dk_resp = kms_client.generate_data_key(KeyId=KMS_KEY_ID, KeySpec="AES_256") - data_key = dk_resp["Plaintext"] - - # Local AES-GCM encrypt - aesgcm = AESGCM(data_key) - nonce = os.urandom(12) - ciphertext = aesgcm.encrypt(nonce, payload, None) - - # Upload ciphertext to S3 - plain_s3.put_object(Bucket=BUCKET, Key=key, Body=nonce + ciphertext) + _record(f"s3ec_roundtrip_{label}_{size_mb}mb", size_mb, s3ec_durations) - # Download and decrypt - resp = plain_s3.get_object(Bucket=BUCKET, Key=key) - blob = resp["Body"].read() - aesgcm.decrypt(blob[:12], blob[12:], None) + # Local crypto + plain S3 roundtrip (only once per size) + if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: + local_durations = [] + for _ in range(NUM_ROUNDS): + key = _unique_key(f"local-rt-{size_mb}mb-") + t0 = time.perf_counter() - local_durations.append(time.perf_counter() - t0) - _record(f"local_crypto_roundtrip_{size_mb}mb", size_mb, local_durations) + dk_resp = kms_client.generate_data_key(KeyId=KMS_KEY_ID, KeySpec="AES_256") + data_key = dk_resp["Plaintext"] + aesgcm = AESGCM(data_key) + nonce = os.urandom(12) + ciphertext = aesgcm.encrypt(nonce, payload, None) -# --------------------------------------------------------------------------- -# Per-algorithm-suite comparison -# --------------------------------------------------------------------------- - + plain_s3.put_object(Bucket=BUCKET, Key=key, Body=nonce + ciphertext) -@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) -@pytest.mark.parametrize("size_mb", OBJECT_SIZES_MB) -def test_s3ec_put_by_algorithm(size_mb, algorithm_suite, commitment_policy): - """Benchmark S3EC put_object for each algorithm suite.""" - payload = _generate_payload(size_mb) - suite_label = algorithm_suite.name.lower() - s3ec = _make_s3ec(algorithm_suite, commitment_policy) - - durations = [] - for _ in range(NUM_ROUNDS): - key = _unique_key(f"alg-put-{suite_label}-{size_mb}mb-") - t0 = time.perf_counter() - s3ec.put_object(Bucket=BUCKET, Key=key, Body=payload) - durations.append(time.perf_counter() - t0) - _record(f"s3ec_put_{suite_label}_{size_mb}mb", size_mb, durations) + resp = plain_s3.get_object(Bucket=BUCKET, Key=key) + blob = resp["Body"].read() + aesgcm.decrypt(blob[:12], blob[12:], None) - -@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) -@pytest.mark.parametrize("size_mb", OBJECT_SIZES_MB) -def test_s3ec_get_by_algorithm(size_mb, algorithm_suite, commitment_policy): - """Benchmark S3EC get_object for each algorithm suite.""" - payload = _generate_payload(size_mb) - suite_label = algorithm_suite.name.lower() - s3ec = _make_s3ec(algorithm_suite, commitment_policy) - - # Upload once - src_key = _unique_key(f"alg-get-src-{suite_label}-{size_mb}mb-") - s3ec.put_object(Bucket=BUCKET, Key=src_key, Body=payload) - - durations = [] - for _ in range(NUM_ROUNDS): - t0 = time.perf_counter() - resp = s3ec.get_object(Bucket=BUCKET, Key=src_key) - resp["Body"].read() - durations.append(time.perf_counter() - t0) - _record(f"s3ec_get_{suite_label}_{size_mb}mb", size_mb, durations) + local_durations.append(time.perf_counter() - t0) + _record(f"local_crypto_roundtrip_{size_mb}mb", size_mb, local_durations) # --------------------------------------------------------------------------- @@ -234,14 +201,18 @@ def test_zz_write_results(): """Final test that writes collected results to a JSON file for the HTML report.""" os.makedirs(os.path.dirname(RESULTS_FILE) or ".", exist_ok=True) with open(RESULTS_FILE, "w") as f: - json.dump({ - "timestamp": datetime.now().isoformat(), - "config": { - "num_rounds": NUM_ROUNDS, - "object_sizes_mb": OBJECT_SIZES_MB, - "bucket": BUCKET, - "region": REGION, + json.dump( + { + "timestamp": datetime.now().isoformat(), + "config": { + "num_rounds": NUM_ROUNDS, + "object_sizes_mb": OBJECT_SIZES_MB, + "bucket": BUCKET, + "region": REGION, + }, + "results": _results, }, - "results": _results, - }, f, indent=2) + f, + indent=2, + ) print(f"\nPerformance results written to {RESULTS_FILE}") From f181d3029d6d1e9f7ea1eab02939a6a248f4fe7b Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Tue, 7 Apr 2026 18:01:36 -0700 Subject: [PATCH 3/9] more rounds --- .github/workflows/python-perf.yml | 2 +- test/performance/conftest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-perf.yml b/.github/workflows/python-perf.yml index 2e6bee40..b22a0358 100644 --- a/.github/workflows/python-perf.yml +++ b/.github/workflows/python-perf.yml @@ -52,7 +52,7 @@ jobs: env: CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} - PERF_NUM_ROUNDS: "5" + PERF_NUM_ROUNDS: "30" - name: Generate performance HTML report if: always() diff --git a/test/performance/conftest.py b/test/performance/conftest.py index e9843ff2..ff4b3eab 100644 --- a/test/performance/conftest.py +++ b/test/performance/conftest.py @@ -18,7 +18,7 @@ ) # Performance test configuration -NUM_ROUNDS = int(os.environ.get("PERF_NUM_ROUNDS", "5")) +NUM_ROUNDS = int(os.environ.get("PERF_NUM_ROUNDS", "30")) OBJECT_SIZES_MB = [10, 25, 50] From ef8f025eef87048de0a32ad62dd2dbde30ff5e59 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 8 Apr 2026 00:14:01 -0700 Subject: [PATCH 4/9] warmup rounds --- test/performance/test_perf_s3_encryption.py | 68 +++++++++++++++++---- 1 file changed, 56 insertions(+), 12 deletions(-) diff --git a/test/performance/test_perf_s3_encryption.py b/test/performance/test_perf_s3_encryption.py index b189eca6..43c66120 100644 --- a/test/performance/test_perf_s3_encryption.py +++ b/test/performance/test_perf_s3_encryption.py @@ -5,6 +5,10 @@ Each test runs multiple rounds with large objects to get a meaningful signal. Results are collected via a module-scoped list and written to a JSON file that the HTML report generator consumes. + +To avoid ordering bias (first algorithm suite paying cold-start penalties for +TCP/TLS connection establishment, KMS client init, etc.), every benchmark +function runs warmup rounds that are discarded before recording. """ import json @@ -21,6 +25,7 @@ PERF_KEY_PREFIX = "perf-test/" RESULTS_FILE = os.environ.get("PERF_RESULTS_FILE", "perf-results/results.json") +WARMUP_ROUNDS = int(os.environ.get("PERF_WARMUP_ROUNDS", "3")) # Collect all benchmark results here _results: list[dict] = [] @@ -50,17 +55,15 @@ def _unique_key(prefix: str) -> str: def _record(test_name, size_mb, durations): - _results.append( - { - "test": test_name, - "size_mb": size_mb, - "rounds": len(durations), - "durations_s": durations, - "mean_s": sum(durations) / len(durations), - "min_s": min(durations), - "max_s": max(durations), - } - ) + _results.append({ + "test": test_name, + "size_mb": size_mb, + "rounds": len(durations), + "durations_s": durations, + "mean_s": sum(durations) / len(durations), + "min_s": min(durations), + "max_s": max(durations), + }) def _algo_label(algorithm_suite): @@ -70,6 +73,23 @@ def _algo_label(algorithm_suite): return "kc_gcm" +def _warmup_put(client_or_s3ec, payload, prefix, is_s3ec=True): + """Run warmup put_object calls to establish connections; results discarded.""" + for _ in range(WARMUP_ROUNDS): + key = _unique_key(f"warmup-{prefix}-") + if is_s3ec: + client_or_s3ec.put_object(Bucket=BUCKET, Key=key, Body=payload) + else: + client_or_s3ec.put_object(Bucket=BUCKET, Key=key, Body=payload) + + +def _warmup_get(client_or_s3ec, object_key): + """Run warmup get_object calls; results discarded.""" + for _ in range(WARMUP_ROUNDS): + resp = client_or_s3ec.get_object(Bucket=BUCKET, Key=object_key) + resp["Body"].read() + + # --------------------------------------------------------------------------- # S3EC put_object vs plain S3 put_object (per algorithm suite) # --------------------------------------------------------------------------- @@ -84,6 +104,7 @@ def test_s3ec_put_vs_plain_put(plain_s3, size_mb, algorithm_suite, commitment_po # Benchmark plain S3 put (only record once per size, skip for second algo) if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: + _warmup_put(plain_s3, payload, f"plain-put-{size_mb}mb", is_s3ec=False) plain_durations = [] for _ in range(NUM_ROUNDS): key = _unique_key(f"plain-put-{size_mb}mb-") @@ -94,6 +115,7 @@ def test_s3ec_put_vs_plain_put(plain_s3, size_mb, algorithm_suite, commitment_po # Benchmark S3EC put s3ec = _make_s3ec(algorithm_suite, commitment_policy) + _warmup_put(s3ec, payload, f"s3ec-put-{label}-{size_mb}mb") s3ec_durations = [] for _ in range(NUM_ROUNDS): key = _unique_key(f"s3ec-put-{label}-{size_mb}mb-") @@ -120,6 +142,7 @@ def test_s3ec_get_vs_plain_get(plain_s3, size_mb, algorithm_suite, commitment_po plain_key = _unique_key(f"plain-get-src-{size_mb}mb-") plain_s3.put_object(Bucket=BUCKET, Key=plain_key, Body=payload) + _warmup_get(plain_s3, plain_key) plain_durations = [] for _ in range(NUM_ROUNDS): t0 = time.perf_counter() @@ -133,6 +156,7 @@ def test_s3ec_get_vs_plain_get(plain_s3, size_mb, algorithm_suite, commitment_po enc_key = _unique_key(f"s3ec-get-src-{label}-{size_mb}mb-") s3ec.put_object(Bucket=BUCKET, Key=enc_key, Body=payload) + _warmup_get(s3ec, enc_key) s3ec_durations = [] for _ in range(NUM_ROUNDS): t0 = time.perf_counter() @@ -156,8 +180,15 @@ def test_s3ec_roundtrip_vs_local_crypto( label = _algo_label(algorithm_suite) payload = _generate_payload(size_mb) - # S3EC roundtrip + # S3EC roundtrip — warmup s3ec = _make_s3ec(algorithm_suite, commitment_policy) + for _ in range(WARMUP_ROUNDS): + wkey = _unique_key(f"warmup-rt-{label}-{size_mb}mb-") + s3ec.put_object(Bucket=BUCKET, Key=wkey, Body=payload) + resp = s3ec.get_object(Bucket=BUCKET, Key=wkey) + resp["Body"].read() + + # S3EC roundtrip — measured s3ec_durations = [] for _ in range(NUM_ROUNDS): key = _unique_key(f"s3ec-rt-{label}-{size_mb}mb-") @@ -170,6 +201,19 @@ def test_s3ec_roundtrip_vs_local_crypto( # Local crypto + plain S3 roundtrip (only once per size) if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: + # Warmup local crypto path + for _ in range(WARMUP_ROUNDS): + wkey = _unique_key(f"warmup-local-rt-{size_mb}mb-") + dk_resp = kms_client.generate_data_key(KeyId=KMS_KEY_ID, KeySpec="AES_256") + aesgcm = AESGCM(dk_resp["Plaintext"]) + nonce = os.urandom(12) + ct = aesgcm.encrypt(nonce, payload, None) + plain_s3.put_object(Bucket=BUCKET, Key=wkey, Body=nonce + ct) + resp = plain_s3.get_object(Bucket=BUCKET, Key=wkey) + blob = resp["Body"].read() + aesgcm.decrypt(blob[:12], blob[12:], None) + + # Measured local_durations = [] for _ in range(NUM_ROUNDS): key = _unique_key(f"local-rt-{size_mb}mb-") From 0af34a21be5cd54bf9143f08717b52679499af5e Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 8 Apr 2026 11:03:31 -0700 Subject: [PATCH 5/9] the revisions will continue until results improve --- test/performance/generate_report.py | 347 +++++++++++++------- test/performance/test_perf_s3_encryption.py | 108 +++--- 2 files changed, 280 insertions(+), 175 deletions(-) diff --git a/test/performance/generate_report.py b/test/performance/generate_report.py index 26858483..14030657 100644 --- a/test/performance/generate_report.py +++ b/test/performance/generate_report.py @@ -1,16 +1,16 @@ #!/usr/bin/env python3 # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Generate an HTML performance report with tables and SVG bar charts.""" +"""Generate an HTML performance report with tables, bar charts, and histograms.""" import json +import math import sys from pathlib import Path RESULTS_FILE = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("perf-results/results.json") OUTPUT_FILE = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("perf-results/report.html") -# Chart palette COLORS = { "plain": "#36a2eb", "aes_gcm": "#ff6384", @@ -25,8 +25,27 @@ def _fmt(seconds: float) -> str: return f"{seconds:.2f} s" +def _percentile(sorted_vals, p): + """Compute the p-th percentile from a sorted list.""" + k = (len(sorted_vals) - 1) * (p / 100) + f = math.floor(k) + c = math.ceil(k) + if f == c: + return sorted_vals[int(k)] + return sorted_vals[f] * (c - k) + sorted_vals[c] * (k - f) + + +def _median(vals): + s = sorted(vals) + return _percentile(s, 50) + + +def _p95(vals): + s = sorted(vals) + return _percentile(s, 95) + + def _lookup(results, prefix, size_mb): - """Find a result entry matching prefix and size.""" for r in results: if r["test"].startswith(prefix) and r["size_mb"] == size_mb: return r @@ -34,140 +53,232 @@ def _lookup(results, prefix, size_mb): def _bar_chart_svg(chart_id, title, groups, sizes, width=700, bar_h=28, gap=6): - """Render a grouped horizontal bar chart as an SVG string. - - Args: - chart_id: unique id for the SVG element - title: chart title - groups: list of dicts {label, color, values: {size_mb: mean_s}} - sizes: list of size_mb values - width: total SVG width - bar_h: height of each bar - gap: vertical gap between bars - """ - label_col_w = 120 # left column for size labels - chart_w = width - label_col_w - 80 # room for value labels on right + """Render a grouped horizontal bar chart (median values) as SVG.""" + label_col_w = 120 + chart_w = width - label_col_w - 80 n_groups = len(groups) - block_h = n_groups * (bar_h + gap) + 20 # per size block - total_h = len(sizes) * block_h + 60 # extra for title + legend + block_h = n_groups * (bar_h + gap) + 20 + total_h = len(sizes) * block_h + 60 - # Find max value for scaling - max_val = 0 - for g in groups: - for v in g["values"].values(): - max_val = max(max_val, v) + max_val = max( + (v for g in groups for v in g["values"].values()), + default=1, + ) if max_val == 0: max_val = 1 - svg_parts = [ + svg = [ f'', + f'xmlns="http://www.w3.org/2000/svg" style="font-family:sans-serif;margin:1rem 0">', f'{title}', ] - - # Legend lx = label_col_w for g in groups: - svg_parts.append( - f'' - ) - svg_parts.append( - f'{g["label"]}' - ) + svg.append(f'') + svg.append(f'{g["label"]}') lx += len(g["label"]) * 7 + 30 - y_offset = 58 + y = 58 for size in sizes: - # Size label - svg_parts.append( - f'{size} MB' ) for i, g in enumerate(groups): val = g["values"].get(size, 0) - bar_w = max(2, (val / max_val) * chart_w) - by = y_offset + i * (bar_h + gap) - svg_parts.append( - f'' ) - svg_parts.append( - f'{_fmt(val)}' ) - y_offset += block_h + y += block_h + svg.append("") + return "\n".join(svg) - svg_parts.append("") - return "\n".join(svg_parts) +def _histogram_svg(chart_id, title, series_list, width=700, height=220, n_bins=15): + """Render overlaid histograms for multiple series as SVG. -def _build_charts(results, sizes): - """Build all SVG charts from the results data.""" - charts = [] - - # --- Chart 1: Put — Plain S3 vs S3EC (AES_GCM) vs S3EC (KC_GCM) --- - put_groups = [ - {"label": "Plain S3", "color": COLORS["plain"], "values": {}}, - {"label": "S3EC AES_GCM", "color": COLORS["aes_gcm"], "values": {}}, - {"label": "S3EC KC_GCM", "color": COLORS["kc_gcm"], "values": {}}, - ] - for s in sizes: - r = _lookup(results, f"plain_s3_put_{s}mb", s) - if r: - put_groups[0]["values"][s] = r["mean_s"] - r = _lookup(results, f"s3ec_put_aes_gcm_{s}mb", s) - if r: - put_groups[1]["values"][s] = r["mean_s"] - r = _lookup(results, f"s3ec_put_kc_gcm_{s}mb", s) - if r: - put_groups[2]["values"][s] = r["mean_s"] - charts.append(_bar_chart_svg("chart-put", "PutObject: Plain S3 vs S3EC", put_groups, sizes)) - - # --- Chart 2: Get — Plain S3 vs S3EC (AES_GCM) vs S3EC (KC_GCM) --- - get_groups = [ - {"label": "Plain S3", "color": COLORS["plain"], "values": {}}, - {"label": "S3EC AES_GCM", "color": COLORS["aes_gcm"], "values": {}}, - {"label": "S3EC KC_GCM", "color": COLORS["kc_gcm"], "values": {}}, - ] - for s in sizes: - r = _lookup(results, f"plain_s3_get_{s}mb", s) - if r: - get_groups[0]["values"][s] = r["mean_s"] - r = _lookup(results, f"s3ec_get_aes_gcm_{s}mb", s) - if r: - get_groups[1]["values"][s] = r["mean_s"] - r = _lookup(results, f"s3ec_get_kc_gcm_{s}mb", s) - if r: - get_groups[2]["values"][s] = r["mean_s"] - charts.append(_bar_chart_svg("chart-get", "GetObject: Plain S3 vs S3EC", get_groups, sizes)) - - # --- Chart 3: Roundtrip — S3EC (AES_GCM) vs S3EC (KC_GCM) vs Local Crypto + Plain S3 --- - rt_groups = [ - {"label": "Local Crypto + Plain S3", "color": COLORS["local"], "values": {}}, - {"label": "S3EC AES_GCM", "color": COLORS["aes_gcm"], "values": {}}, - {"label": "S3EC KC_GCM", "color": COLORS["kc_gcm"], "values": {}}, + Args: + chart_id: unique SVG id + title: chart title + series_list: list of {label, color, durations: [float]} + width, height: SVG dimensions + n_bins: number of histogram bins + """ + # Compute global range across all series + all_vals = [d for s in series_list for d in s["durations"]] + if not all_vals: + return "" + lo = min(all_vals) + hi = max(all_vals) + if lo == hi: + hi = lo + 0.001 # avoid zero-width range + + margin_l, margin_r, margin_t, margin_b = 60, 20, 50, 40 + plot_w = width - margin_l - margin_r + plot_h = height - margin_t - margin_b + bin_width = (hi - lo) / n_bins + + # Build histogram counts for each series + histograms = [] + global_max_count = 0 + for s in series_list: + counts = [0] * n_bins + for d in s["durations"]: + idx = min(int((d - lo) / bin_width), n_bins - 1) + counts[idx] += 1 + global_max_count = max(global_max_count, max(counts)) + histograms.append(counts) + if global_max_count == 0: + global_max_count = 1 + + svg = [ + f'', + f'{title}', ] - for s in sizes: - r = _lookup(results, f"local_crypto_roundtrip_{s}mb", s) - if r: - rt_groups[0]["values"][s] = r["mean_s"] - r = _lookup(results, f"s3ec_roundtrip_aes_gcm_{s}mb", s) - if r: - rt_groups[1]["values"][s] = r["mean_s"] - r = _lookup(results, f"s3ec_roundtrip_kc_gcm_{s}mb", s) - if r: - rt_groups[2]["values"][s] = r["mean_s"] - charts.append( - _bar_chart_svg("chart-rt", "Roundtrip: S3EC vs Local Crypto + Plain S3", rt_groups, sizes) + + # Legend + lx = margin_l + for s in series_list: + svg.append(f'') + svg.append(f'{s["label"]}') + lx += len(s["label"]) * 6 + 24 + + # Axes + ax_y = margin_t + plot_h + svg.append( + f'' ) + svg.append( + f'' + ) + + # X-axis labels (5 ticks) + for i in range(6): + val = lo + (hi - lo) * i / 5 + x = margin_l + plot_w * i / 5 + svg.append( + f'{_fmt(val)}' + ) + + # Y-axis labels + for i in range(4): + cnt = int(global_max_count * i / 3) + y_pos = ax_y - plot_h * i / 3 + svg.append( + f'{cnt}' + ) + + # Draw bars for each series (slightly offset for overlap visibility) + bar_px = plot_w / n_bins + n_series = len(series_list) + sub_w = bar_px / n_series if n_series > 1 else bar_px * 0.8 + + for si, (s, counts) in enumerate(zip(series_list, histograms)): + for bi, cnt in enumerate(counts): + if cnt == 0: + continue + bh = (cnt / global_max_count) * plot_h + bx = margin_l + bi * bar_px + si * sub_w + by = ax_y - bh + svg.append( + f'' + ) + + svg.append("") + return "\n".join(svg) + + +def _build_charts_and_histograms(results, sizes): + """Build bar charts (median) and histograms for each category.""" + html_parts = [] + + # --- Define chart groups --- + chart_defs = [ + { + "id": "put", + "title": "PutObject: Plain S3 vs S3EC", + "series": [ + ("Plain S3", "plain", "plain_s3_put"), + ("S3EC AES_GCM", "aes_gcm", "s3ec_put_aes_gcm"), + ("S3EC KC_GCM", "kc_gcm", "s3ec_put_kc_gcm"), + ], + }, + { + "id": "get", + "title": "GetObject: Plain S3 vs S3EC", + "series": [ + ("Plain S3", "plain", "plain_s3_get"), + ("S3EC AES_GCM", "aes_gcm", "s3ec_get_aes_gcm"), + ("S3EC KC_GCM", "kc_gcm", "s3ec_get_kc_gcm"), + ], + }, + { + "id": "rt", + "title": "Roundtrip: S3EC vs Local Crypto + Plain S3", + "series": [ + ("Local Crypto + Plain S3", "local", "local_crypto_roundtrip"), + ("S3EC AES_GCM", "aes_gcm", "s3ec_roundtrip_aes_gcm"), + ("S3EC KC_GCM", "kc_gcm", "s3ec_roundtrip_kc_gcm"), + ], + }, + ] + + for cdef in chart_defs: + # Bar chart using median + groups = [] + for label, color_key, prefix in cdef["series"]: + vals = {} + for s in sizes: + r = _lookup(results, f"{prefix}_{s}mb", s) + if r: + vals[s] = _median(r["durations_s"]) + groups.append({"label": label, "color": COLORS[color_key], "values": vals}) + html_parts.append( + _bar_chart_svg(f"chart-{cdef['id']}", f"{cdef['title']} (Median)", groups, sizes) + ) - return "\n".join(charts) + # Histograms — one per payload size, stacked vertically + for s in sizes: + series_list = [] + for label, color_key, prefix in cdef["series"]: + r = _lookup(results, f"{prefix}_{s}mb", s) + if r: + series_list.append( + { + "label": label, + "color": COLORS[color_key], + "durations": r["durations_s"], + } + ) + if series_list: + html_parts.append( + _histogram_svg( + f"hist-{cdef['id']}-{s}mb", + f"{cdef['title']} — {s} MB Distribution", + series_list, + ) + ) + + return "\n".join(html_parts) def _build_table(results): - """Build the full results table HTML.""" - # Group results by category + """Build the full results table with median and p95.""" groups: dict[str, list[dict]] = {} for r in results: name = r["test"] @@ -185,13 +296,18 @@ def _build_table(results): for cat, items in groups.items(): rows = "" for r in sorted(items, key=lambda x: (x["size_mb"], x["test"])): - durations_str = ", ".join(_fmt(d) for d in r["durations_s"]) + d = r["durations_s"] + med = _median(d) + p95 = _p95(d) + durations_str = ", ".join(_fmt(v) for v in d) rows += f""" {r['test']} {r['size_mb']} MB {r['rounds']} + {_fmt(med)} {_fmt(r['mean_s'])} + {_fmt(p95)} {_fmt(r['min_s'])} {_fmt(r['max_s'])} {durations_str} @@ -203,7 +319,8 @@ def _build_table(results): TestSizeRounds - MeanMinMaxAll Durations + MedianMeanp95 + MinMaxAll Durations {rows} @@ -218,7 +335,7 @@ def generate_html(data: dict) -> str: timestamp = data["timestamp"] sizes = config["object_sizes_mb"] - charts_html = _build_charts(results, sizes) + visuals_html = _build_charts_and_histograms(results, sizes) tables_html = _build_table(results) return f""" @@ -232,9 +349,10 @@ def generate_html(data: dict) -> str: h1 {{ color: #232f3e; border-bottom: 3px solid #ff9900; padding-bottom: 0.5rem; }} h2 {{ color: #232f3e; margin-top: 2rem; }} .meta {{ color: #555; font-size: 0.9rem; margin-bottom: 1.5rem; }} - .charts {{ display: flex; flex-wrap: wrap; gap: 2rem; margin-bottom: 2rem; }} - .charts svg {{ background: #fff; border-radius: 6px; - box-shadow: 0 1px 3px rgba(0,0,0,0.1); padding: 0.5rem; }} + .visuals {{ margin-bottom: 2rem; }} + .visuals svg {{ background: #fff; border-radius: 6px; + box-shadow: 0 1px 3px rgba(0,0,0,0.1); padding: 0.5rem; + display: block; margin-bottom: 0.5rem; }} table {{ border-collapse: collapse; width: 100%; margin-bottom: 1.5rem; background: #fff; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }} th {{ background: #232f3e; color: #fff; padding: 0.6rem 0.8rem; @@ -253,9 +371,8 @@ def generate_html(data: dict) -> str: Bucket: {config['bucket']} · Region: {config['region']} -

Charts

-
-{charts_html} +
+{visuals_html}
{tables_html} diff --git a/test/performance/test_perf_s3_encryption.py b/test/performance/test_perf_s3_encryption.py index 43c66120..94fc6b77 100644 --- a/test/performance/test_perf_s3_encryption.py +++ b/test/performance/test_perf_s3_encryption.py @@ -6,9 +6,8 @@ Results are collected via a module-scoped list and written to a JSON file that the HTML report generator consumes. -To avoid ordering bias (first algorithm suite paying cold-start penalties for -TCP/TLS connection establishment, KMS client init, etc.), every benchmark -function runs warmup rounds that are discarded before recording. +Connection warmup is handled once at module level with a small payload so that +TCP/TLS establishment and KMS client init costs don't pollute any benchmark. """ import json @@ -25,7 +24,6 @@ PERF_KEY_PREFIX = "perf-test/" RESULTS_FILE = os.environ.get("PERF_RESULTS_FILE", "perf-results/results.json") -WARMUP_ROUNDS = int(os.environ.get("PERF_WARMUP_ROUNDS", "3")) # Collect all benchmark results here _results: list[dict] = [] @@ -43,11 +41,19 @@ ), ] +# Pre-generate payloads once at module level to avoid repeated large allocations +_PAYLOADS: dict[int, bytes] = {} -def _generate_payload(size_mb: int) -> bytes: - """Generate a random payload of the given size in MB.""" - chunk = os.urandom(1024) # 1 KB random chunk - return (chunk * 1024 * size_mb)[: size_mb * 1024 * 1024] + +def _get_payload(size_mb: int) -> bytes: + if size_mb not in _PAYLOADS: + chunk = os.urandom(1024) + _PAYLOADS[size_mb] = (chunk * 1024 * size_mb)[: size_mb * 1024 * 1024] + return _PAYLOADS[size_mb] + + +# Small payload used only for connection warmup (1 KB) +_WARMUP_PAYLOAD = b"x" * 1024 def _unique_key(prefix: str) -> str: @@ -55,15 +61,17 @@ def _unique_key(prefix: str) -> str: def _record(test_name, size_mb, durations): - _results.append({ - "test": test_name, - "size_mb": size_mb, - "rounds": len(durations), - "durations_s": durations, - "mean_s": sum(durations) / len(durations), - "min_s": min(durations), - "max_s": max(durations), - }) + _results.append( + { + "test": test_name, + "size_mb": size_mb, + "rounds": len(durations), + "durations_s": durations, + "mean_s": sum(durations) / len(durations), + "min_s": min(durations), + "max_s": max(durations), + } + ) def _algo_label(algorithm_suite): @@ -73,21 +81,16 @@ def _algo_label(algorithm_suite): return "kc_gcm" -def _warmup_put(client_or_s3ec, payload, prefix, is_s3ec=True): - """Run warmup put_object calls to establish connections; results discarded.""" - for _ in range(WARMUP_ROUNDS): - key = _unique_key(f"warmup-{prefix}-") - if is_s3ec: - client_or_s3ec.put_object(Bucket=BUCKET, Key=key, Body=payload) - else: - client_or_s3ec.put_object(Bucket=BUCKET, Key=key, Body=payload) - - -def _warmup_get(client_or_s3ec, object_key): - """Run warmup get_object calls; results discarded.""" - for _ in range(WARMUP_ROUNDS): - resp = client_or_s3ec.get_object(Bucket=BUCKET, Key=object_key) - resp["Body"].read() +def _warmup_connection(client, is_plain=False): + """Warm up TCP/TLS connections with a tiny payload. Called once per client.""" + key = _unique_key("warmup-conn-") + if is_plain: + client.put_object(Bucket=BUCKET, Key=key, Body=_WARMUP_PAYLOAD) + resp = client.get_object(Bucket=BUCKET, Key=key) + else: + client.put_object(Bucket=BUCKET, Key=key, Body=_WARMUP_PAYLOAD) + resp = client.get_object(Bucket=BUCKET, Key=key) + resp["Body"].read() # --------------------------------------------------------------------------- @@ -100,11 +103,11 @@ def _warmup_get(client_or_s3ec, object_key): def test_s3ec_put_vs_plain_put(plain_s3, size_mb, algorithm_suite, commitment_policy): """Compare S3EC put_object latency against plain S3 put_object.""" label = _algo_label(algorithm_suite) - payload = _generate_payload(size_mb) + payload = _get_payload(size_mb) - # Benchmark plain S3 put (only record once per size, skip for second algo) + # Benchmark plain S3 put (only record once per size) if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: - _warmup_put(plain_s3, payload, f"plain-put-{size_mb}mb", is_s3ec=False) + _warmup_connection(plain_s3, is_plain=True) plain_durations = [] for _ in range(NUM_ROUNDS): key = _unique_key(f"plain-put-{size_mb}mb-") @@ -115,7 +118,7 @@ def test_s3ec_put_vs_plain_put(plain_s3, size_mb, algorithm_suite, commitment_po # Benchmark S3EC put s3ec = _make_s3ec(algorithm_suite, commitment_policy) - _warmup_put(s3ec, payload, f"s3ec-put-{label}-{size_mb}mb") + _warmup_connection(s3ec) s3ec_durations = [] for _ in range(NUM_ROUNDS): key = _unique_key(f"s3ec-put-{label}-{size_mb}mb-") @@ -135,14 +138,14 @@ def test_s3ec_put_vs_plain_put(plain_s3, size_mb, algorithm_suite, commitment_po def test_s3ec_get_vs_plain_get(plain_s3, size_mb, algorithm_suite, commitment_policy): """Compare S3EC get_object latency against plain S3 get_object.""" label = _algo_label(algorithm_suite) - payload = _generate_payload(size_mb) + payload = _get_payload(size_mb) # Upload a plain object (only once per size) if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: plain_key = _unique_key(f"plain-get-src-{size_mb}mb-") plain_s3.put_object(Bucket=BUCKET, Key=plain_key, Body=payload) - _warmup_get(plain_s3, plain_key) + _warmup_connection(plain_s3, is_plain=True) plain_durations = [] for _ in range(NUM_ROUNDS): t0 = time.perf_counter() @@ -156,7 +159,7 @@ def test_s3ec_get_vs_plain_get(plain_s3, size_mb, algorithm_suite, commitment_po enc_key = _unique_key(f"s3ec-get-src-{label}-{size_mb}mb-") s3ec.put_object(Bucket=BUCKET, Key=enc_key, Body=payload) - _warmup_get(s3ec, enc_key) + _warmup_connection(s3ec) s3ec_durations = [] for _ in range(NUM_ROUNDS): t0 = time.perf_counter() @@ -178,17 +181,11 @@ def test_s3ec_roundtrip_vs_local_crypto( ): """Compare S3EC roundtrip against manual local AES-GCM encrypt + plain S3 roundtrip.""" label = _algo_label(algorithm_suite) - payload = _generate_payload(size_mb) + payload = _get_payload(size_mb) - # S3EC roundtrip — warmup + # S3EC roundtrip s3ec = _make_s3ec(algorithm_suite, commitment_policy) - for _ in range(WARMUP_ROUNDS): - wkey = _unique_key(f"warmup-rt-{label}-{size_mb}mb-") - s3ec.put_object(Bucket=BUCKET, Key=wkey, Body=payload) - resp = s3ec.get_object(Bucket=BUCKET, Key=wkey) - resp["Body"].read() - - # S3EC roundtrip — measured + _warmup_connection(s3ec) s3ec_durations = [] for _ in range(NUM_ROUNDS): key = _unique_key(f"s3ec-rt-{label}-{size_mb}mb-") @@ -201,19 +198,10 @@ def test_s3ec_roundtrip_vs_local_crypto( # Local crypto + plain S3 roundtrip (only once per size) if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: - # Warmup local crypto path - for _ in range(WARMUP_ROUNDS): - wkey = _unique_key(f"warmup-local-rt-{size_mb}mb-") - dk_resp = kms_client.generate_data_key(KeyId=KMS_KEY_ID, KeySpec="AES_256") - aesgcm = AESGCM(dk_resp["Plaintext"]) - nonce = os.urandom(12) - ct = aesgcm.encrypt(nonce, payload, None) - plain_s3.put_object(Bucket=BUCKET, Key=wkey, Body=nonce + ct) - resp = plain_s3.get_object(Bucket=BUCKET, Key=wkey) - blob = resp["Body"].read() - aesgcm.decrypt(blob[:12], blob[12:], None) + _warmup_connection(plain_s3, is_plain=True) + # Warm up KMS connection + kms_client.generate_data_key(KeyId=KMS_KEY_ID, KeySpec="AES_256") - # Measured local_durations = [] for _ in range(NUM_ROUNDS): key = _unique_key(f"local-rt-{size_mb}mb-") From 21dc40f66601fbc6b0e1d6e27ea968de92322192 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 17 Apr 2026 12:27:07 -0700 Subject: [PATCH 6/9] shuffle --- test/performance/conftest.py | 17 -- test/performance/test_perf_s3_encryption.py | 307 +++++++++++--------- 2 files changed, 163 insertions(+), 161 deletions(-) diff --git a/test/performance/conftest.py b/test/performance/conftest.py index ff4b3eab..2e686a30 100644 --- a/test/performance/conftest.py +++ b/test/performance/conftest.py @@ -9,7 +9,6 @@ from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.materials.kms_keyring import KmsKeyring -from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy BUCKET = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") REGION = os.environ.get("CI_AWS_REGION", "us-west-2") @@ -34,22 +33,6 @@ def _make_s3ec(algorithm_suite, commitment_policy): return S3EncryptionClient(wrapped_client, config) -@pytest.fixture(scope="module") -def s3ec_v2(): - return _make_s3ec( - AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, - CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, - ) - - -@pytest.fixture(scope="module") -def s3ec_v3(): - return _make_s3ec( - AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, - ) - - @pytest.fixture(scope="module") def plain_s3(): return boto3.client("s3", region_name=REGION) diff --git a/test/performance/test_perf_s3_encryption.py b/test/performance/test_perf_s3_encryption.py index 94fc6b77..b6ae067a 100644 --- a/test/performance/test_perf_s3_encryption.py +++ b/test/performance/test_perf_s3_encryption.py @@ -3,15 +3,17 @@ """Performance tests comparing S3EC against plaintext S3 and local encryption + S3 upload. Each test runs multiple rounds with large objects to get a meaningful signal. +To control for temporal network variation, all variants within a test are +interleaved: round N of every variant runs back-to-back before moving to +round N+1. This ensures each variant experiences the same network conditions. + Results are collected via a module-scoped list and written to a JSON file that the HTML report generator consumes. - -Connection warmup is handled once at module level with a small payload so that -TCP/TLS establishment and KMS client init costs don't pollute any benchmark. """ import json import os +import random import time from datetime import datetime @@ -25,24 +27,15 @@ PERF_KEY_PREFIX = "perf-test/" RESULTS_FILE = os.environ.get("PERF_RESULTS_FILE", "perf-results/results.json") -# Collect all benchmark results here _results: list[dict] = [] -ALGORITHM_CONFIGS = [ - pytest.param( - AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, - CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, - id="AES_GCM", - ), - pytest.param( - AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, - id="KC_GCM", - ), -] - -# Pre-generate payloads once at module level to avoid repeated large allocations +# Pre-generate payloads once at module level _PAYLOADS: dict[int, bytes] = {} +_WARMUP_PAYLOAD = b"x" * 1024 + +# Algorithm suite configs +_AES_GCM = (AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) +_KC_GCM = (AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) def _get_payload(size_mb: int) -> bytes: @@ -52,176 +45,202 @@ def _get_payload(size_mb: int) -> bytes: return _PAYLOADS[size_mb] -# Small payload used only for connection warmup (1 KB) -_WARMUP_PAYLOAD = b"x" * 1024 - - def _unique_key(prefix: str) -> str: return PERF_KEY_PREFIX + prefix + datetime.now().strftime("%Y%m%d-%H%M%S-%f") def _record(test_name, size_mb, durations): - _results.append( - { - "test": test_name, - "size_mb": size_mb, - "rounds": len(durations), - "durations_s": durations, - "mean_s": sum(durations) / len(durations), - "min_s": min(durations), - "max_s": max(durations), - } - ) - - -def _algo_label(algorithm_suite): - """Short human-readable label for an algorithm suite.""" - if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: - return "aes_gcm" - return "kc_gcm" - - -def _warmup_connection(client, is_plain=False): - """Warm up TCP/TLS connections with a tiny payload. Called once per client.""" + _results.append({ + "test": test_name, + "size_mb": size_mb, + "rounds": len(durations), + "durations_s": durations, + "mean_s": sum(durations) / len(durations), + "min_s": min(durations), + "max_s": max(durations), + }) + + +def _warmup_connection(client): + """Warm up TCP/TLS connections with a tiny payload.""" key = _unique_key("warmup-conn-") - if is_plain: - client.put_object(Bucket=BUCKET, Key=key, Body=_WARMUP_PAYLOAD) - resp = client.get_object(Bucket=BUCKET, Key=key) - else: - client.put_object(Bucket=BUCKET, Key=key, Body=_WARMUP_PAYLOAD) - resp = client.get_object(Bucket=BUCKET, Key=key) + client.put_object(Bucket=BUCKET, Key=key, Body=_WARMUP_PAYLOAD) + resp = client.get_object(Bucket=BUCKET, Key=key) resp["Body"].read() # --------------------------------------------------------------------------- -# S3EC put_object vs plain S3 put_object (per algorithm suite) +# Interleaved put_object benchmark # --------------------------------------------------------------------------- -@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) @pytest.mark.parametrize("size_mb", OBJECT_SIZES_MB) -def test_s3ec_put_vs_plain_put(plain_s3, size_mb, algorithm_suite, commitment_policy): - """Compare S3EC put_object latency against plain S3 put_object.""" - label = _algo_label(algorithm_suite) +def test_put_interleaved(plain_s3, size_mb): + """Interleaved put_object: plain S3, S3EC AES_GCM, S3EC KC_GCM.""" payload = _get_payload(size_mb) - # Benchmark plain S3 put (only record once per size) - if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: - _warmup_connection(plain_s3, is_plain=True) - plain_durations = [] - for _ in range(NUM_ROUNDS): - key = _unique_key(f"plain-put-{size_mb}mb-") - t0 = time.perf_counter() - plain_s3.put_object(Bucket=BUCKET, Key=key, Body=payload) - plain_durations.append(time.perf_counter() - t0) - _record(f"plain_s3_put_{size_mb}mb", size_mb, plain_durations) - - # Benchmark S3EC put - s3ec = _make_s3ec(algorithm_suite, commitment_policy) - _warmup_connection(s3ec) - s3ec_durations = [] - for _ in range(NUM_ROUNDS): - key = _unique_key(f"s3ec-put-{label}-{size_mb}mb-") + s3ec_aes = _make_s3ec(*_AES_GCM) + s3ec_kc = _make_s3ec(*_KC_GCM) + + # Warm up all connections + _warmup_connection(plain_s3) + _warmup_connection(s3ec_aes) + _warmup_connection(s3ec_kc) + + plain_d, aes_d, kc_d = [], [], [] + + # Define the three variants as callables + def run_plain(): + key = _unique_key(f"plain-put-{size_mb}mb-") t0 = time.perf_counter() - s3ec.put_object(Bucket=BUCKET, Key=key, Body=payload) - s3ec_durations.append(time.perf_counter() - t0) - _record(f"s3ec_put_{label}_{size_mb}mb", size_mb, s3ec_durations) + plain_s3.put_object(Bucket=BUCKET, Key=key, Body=payload) + return time.perf_counter() - t0 + + def run_aes(): + key = _unique_key(f"s3ec-put-aes-{size_mb}mb-") + t0 = time.perf_counter() + s3ec_aes.put_object(Bucket=BUCKET, Key=key, Body=payload) + return time.perf_counter() - t0 + + def run_kc(): + key = _unique_key(f"s3ec-put-kc-{size_mb}mb-") + t0 = time.perf_counter() + s3ec_kc.put_object(Bucket=BUCKET, Key=key, Body=payload) + return time.perf_counter() - t0 + + variants = [(run_plain, plain_d), (run_aes, aes_d), (run_kc, kc_d)] + + for _ in range(NUM_ROUNDS): + # Shuffle order each round to eliminate positional bias + random.shuffle(variants) + for fn, collector in variants: + collector.append(fn()) + + _record(f"plain_s3_put_{size_mb}mb", size_mb, plain_d) + _record(f"s3ec_put_aes_gcm_{size_mb}mb", size_mb, aes_d) + _record(f"s3ec_put_kc_gcm_{size_mb}mb", size_mb, kc_d) # --------------------------------------------------------------------------- -# S3EC get_object vs plain S3 get_object (per algorithm suite) +# Interleaved get_object benchmark # --------------------------------------------------------------------------- -@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) @pytest.mark.parametrize("size_mb", OBJECT_SIZES_MB) -def test_s3ec_get_vs_plain_get(plain_s3, size_mb, algorithm_suite, commitment_policy): - """Compare S3EC get_object latency against plain S3 get_object.""" - label = _algo_label(algorithm_suite) +def test_get_interleaved(plain_s3, size_mb): + """Interleaved get_object: plain S3, S3EC AES_GCM, S3EC KC_GCM.""" payload = _get_payload(size_mb) - # Upload a plain object (only once per size) - if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: - plain_key = _unique_key(f"plain-get-src-{size_mb}mb-") - plain_s3.put_object(Bucket=BUCKET, Key=plain_key, Body=payload) - - _warmup_connection(plain_s3, is_plain=True) - plain_durations = [] - for _ in range(NUM_ROUNDS): - t0 = time.perf_counter() - resp = plain_s3.get_object(Bucket=BUCKET, Key=plain_key) - resp["Body"].read() - plain_durations.append(time.perf_counter() - t0) - _record(f"plain_s3_get_{size_mb}mb", size_mb, plain_durations) - - # Upload an encrypted object and benchmark get - s3ec = _make_s3ec(algorithm_suite, commitment_policy) - enc_key = _unique_key(f"s3ec-get-src-{label}-{size_mb}mb-") - s3ec.put_object(Bucket=BUCKET, Key=enc_key, Body=payload) - - _warmup_connection(s3ec) - s3ec_durations = [] - for _ in range(NUM_ROUNDS): + s3ec_aes = _make_s3ec(*_AES_GCM) + s3ec_kc = _make_s3ec(*_KC_GCM) + + # Upload source objects + plain_key = _unique_key(f"plain-get-src-{size_mb}mb-") + plain_s3.put_object(Bucket=BUCKET, Key=plain_key, Body=payload) + + aes_key = _unique_key(f"s3ec-get-src-aes-{size_mb}mb-") + s3ec_aes.put_object(Bucket=BUCKET, Key=aes_key, Body=payload) + + kc_key = _unique_key(f"s3ec-get-src-kc-{size_mb}mb-") + s3ec_kc.put_object(Bucket=BUCKET, Key=kc_key, Body=payload) + + # Warm up all connections + _warmup_connection(plain_s3) + _warmup_connection(s3ec_aes) + _warmup_connection(s3ec_kc) + + plain_d, aes_d, kc_d = [], [], [] + + def run_plain(): + t0 = time.perf_counter() + resp = plain_s3.get_object(Bucket=BUCKET, Key=plain_key) + resp["Body"].read() + return time.perf_counter() - t0 + + def run_aes(): t0 = time.perf_counter() - resp = s3ec.get_object(Bucket=BUCKET, Key=enc_key) + resp = s3ec_aes.get_object(Bucket=BUCKET, Key=aes_key) resp["Body"].read() - s3ec_durations.append(time.perf_counter() - t0) - _record(f"s3ec_get_{label}_{size_mb}mb", size_mb, s3ec_durations) + return time.perf_counter() - t0 + + def run_kc(): + t0 = time.perf_counter() + resp = s3ec_kc.get_object(Bucket=BUCKET, Key=kc_key) + resp["Body"].read() + return time.perf_counter() - t0 + + variants = [(run_plain, plain_d), (run_aes, aes_d), (run_kc, kc_d)] + + for _ in range(NUM_ROUNDS): + random.shuffle(variants) + for fn, collector in variants: + collector.append(fn()) + + _record(f"plain_s3_get_{size_mb}mb", size_mb, plain_d) + _record(f"s3ec_get_aes_gcm_{size_mb}mb", size_mb, aes_d) + _record(f"s3ec_get_kc_gcm_{size_mb}mb", size_mb, kc_d) # --------------------------------------------------------------------------- -# S3EC roundtrip vs local encrypt + plain S3 upload (per algorithm suite) +# Interleaved roundtrip benchmark # --------------------------------------------------------------------------- -@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) @pytest.mark.parametrize("size_mb", OBJECT_SIZES_MB) -def test_s3ec_roundtrip_vs_local_crypto( - plain_s3, kms_client, size_mb, algorithm_suite, commitment_policy -): - """Compare S3EC roundtrip against manual local AES-GCM encrypt + plain S3 roundtrip.""" - label = _algo_label(algorithm_suite) +def test_roundtrip_interleaved(plain_s3, kms_client, size_mb): + """Interleaved roundtrip: S3EC AES_GCM, S3EC KC_GCM, local crypto + plain S3.""" payload = _get_payload(size_mb) - # S3EC roundtrip - s3ec = _make_s3ec(algorithm_suite, commitment_policy) - _warmup_connection(s3ec) - s3ec_durations = [] - for _ in range(NUM_ROUNDS): - key = _unique_key(f"s3ec-rt-{label}-{size_mb}mb-") - t0 = time.perf_counter() - s3ec.put_object(Bucket=BUCKET, Key=key, Body=payload) - resp = s3ec.get_object(Bucket=BUCKET, Key=key) - resp["Body"].read() - s3ec_durations.append(time.perf_counter() - t0) - _record(f"s3ec_roundtrip_{label}_{size_mb}mb", size_mb, s3ec_durations) + s3ec_aes = _make_s3ec(*_AES_GCM) + s3ec_kc = _make_s3ec(*_KC_GCM) - # Local crypto + plain S3 roundtrip (only once per size) - if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: - _warmup_connection(plain_s3, is_plain=True) - # Warm up KMS connection - kms_client.generate_data_key(KeyId=KMS_KEY_ID, KeySpec="AES_256") + # Warm up all connections + _warmup_connection(plain_s3) + _warmup_connection(s3ec_aes) + _warmup_connection(s3ec_kc) + kms_client.generate_data_key(KeyId=KMS_KEY_ID, KeySpec="AES_256") - local_durations = [] - for _ in range(NUM_ROUNDS): - key = _unique_key(f"local-rt-{size_mb}mb-") - t0 = time.perf_counter() + aes_d, kc_d, local_d = [], [], [] - dk_resp = kms_client.generate_data_key(KeyId=KMS_KEY_ID, KeySpec="AES_256") - data_key = dk_resp["Plaintext"] + def run_aes(): + key = _unique_key(f"s3ec-rt-aes-{size_mb}mb-") + t0 = time.perf_counter() + s3ec_aes.put_object(Bucket=BUCKET, Key=key, Body=payload) + resp = s3ec_aes.get_object(Bucket=BUCKET, Key=key) + resp["Body"].read() + return time.perf_counter() - t0 - aesgcm = AESGCM(data_key) - nonce = os.urandom(12) - ciphertext = aesgcm.encrypt(nonce, payload, None) + def run_kc(): + key = _unique_key(f"s3ec-rt-kc-{size_mb}mb-") + t0 = time.perf_counter() + s3ec_kc.put_object(Bucket=BUCKET, Key=key, Body=payload) + resp = s3ec_kc.get_object(Bucket=BUCKET, Key=key) + resp["Body"].read() + return time.perf_counter() - t0 - plain_s3.put_object(Bucket=BUCKET, Key=key, Body=nonce + ciphertext) + def run_local(): + key = _unique_key(f"local-rt-{size_mb}mb-") + t0 = time.perf_counter() + dk_resp = kms_client.generate_data_key(KeyId=KMS_KEY_ID, KeySpec="AES_256") + aesgcm = AESGCM(dk_resp["Plaintext"]) + nonce = os.urandom(12) + ciphertext = aesgcm.encrypt(nonce, payload, None) + plain_s3.put_object(Bucket=BUCKET, Key=key, Body=nonce + ciphertext) + resp = plain_s3.get_object(Bucket=BUCKET, Key=key) + blob = resp["Body"].read() + aesgcm.decrypt(blob[:12], blob[12:], None) + return time.perf_counter() - t0 + + variants = [(run_aes, aes_d), (run_kc, kc_d), (run_local, local_d)] - resp = plain_s3.get_object(Bucket=BUCKET, Key=key) - blob = resp["Body"].read() - aesgcm.decrypt(blob[:12], blob[12:], None) + for _ in range(NUM_ROUNDS): + random.shuffle(variants) + for fn, collector in variants: + collector.append(fn()) - local_durations.append(time.perf_counter() - t0) - _record(f"local_crypto_roundtrip_{size_mb}mb", size_mb, local_durations) + _record(f"s3ec_roundtrip_aes_gcm_{size_mb}mb", size_mb, aes_d) + _record(f"s3ec_roundtrip_kc_gcm_{size_mb}mb", size_mb, kc_d) + _record(f"local_crypto_roundtrip_{size_mb}mb", size_mb, local_d) # --------------------------------------------------------------------------- From b5aef366c9ccfc631500a102773e0581f0ef37b7 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 17 Apr 2026 13:55:55 -0700 Subject: [PATCH 7/9] format --- test/performance/test_perf_s3_encryption.py | 30 +++++++++++++-------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/test/performance/test_perf_s3_encryption.py b/test/performance/test_perf_s3_encryption.py index b6ae067a..590a0eb2 100644 --- a/test/performance/test_perf_s3_encryption.py +++ b/test/performance/test_perf_s3_encryption.py @@ -34,8 +34,14 @@ _WARMUP_PAYLOAD = b"x" * 1024 # Algorithm suite configs -_AES_GCM = (AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) -_KC_GCM = (AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) +_AES_GCM = ( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, +) +_KC_GCM = ( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, +) def _get_payload(size_mb: int) -> bytes: @@ -50,15 +56,17 @@ def _unique_key(prefix: str) -> str: def _record(test_name, size_mb, durations): - _results.append({ - "test": test_name, - "size_mb": size_mb, - "rounds": len(durations), - "durations_s": durations, - "mean_s": sum(durations) / len(durations), - "min_s": min(durations), - "max_s": max(durations), - }) + _results.append( + { + "test": test_name, + "size_mb": size_mb, + "rounds": len(durations), + "durations_s": durations, + "mean_s": sum(durations) / len(durations), + "min_s": min(durations), + "max_s": max(durations), + } + ) def _warmup_connection(client): From 4d55221e0dddc6503d24a8893f22aa25fcfdb52f Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 17 Apr 2026 16:15:45 -0700 Subject: [PATCH 8/9] I am because we are --- .github/workflows/python-perf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-perf.yml b/.github/workflows/python-perf.yml index b22a0358..c5a4ab41 100644 --- a/.github/workflows/python-perf.yml +++ b/.github/workflows/python-perf.yml @@ -11,7 +11,7 @@ on: jobs: python-perf: - runs-on: macos-14-large + runs-on: ubuntu-latest permissions: id-token: write contents: read From 267ab48256b2b8dcbca708de54c3a15e52a580ed Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 17 Apr 2026 16:31:18 -0700 Subject: [PATCH 9/9] fix test workflow --- .github/workflows/python-integ.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-integ.yml b/.github/workflows/python-integ.yml index 0f50c8e8..ec50a72c 100644 --- a/.github/workflows/python-integ.yml +++ b/.github/workflows/python-integ.yml @@ -49,7 +49,7 @@ jobs: - name: Run unit tests run: | - uv run pytest test/ --ignore=test/integration/ --verbose \ + uv run pytest test/ --ignore=test/integration/ --ignore=test/performance/ --verbose \ --cov=src/s3_encryption --cov-report=term-missing --cov-report=html:coverage-unit \ --cov-fail-under=89