From b67c3cedc4268a6e7de6b2ab79f94c6e1344b191 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Mon, 10 Nov 2025 16:09:35 -0800 Subject: [PATCH 01/45] Latest Ruby and spec --- test-server/ruby-v2-server/local-ruby-sdk | 2 +- test-server/ruby-v3-server/local-ruby-sdk | 2 +- test-server/specification | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test-server/ruby-v2-server/local-ruby-sdk b/test-server/ruby-v2-server/local-ruby-sdk index 9bffb525..582e0241 160000 --- a/test-server/ruby-v2-server/local-ruby-sdk +++ b/test-server/ruby-v2-server/local-ruby-sdk @@ -1 +1 @@ -Subproject commit 9bffb525765b2aeebd6203072bf6e94d2ab53f90 +Subproject commit 582e02418ac2c5540c48dd089a7db506712e6f94 diff --git a/test-server/ruby-v3-server/local-ruby-sdk b/test-server/ruby-v3-server/local-ruby-sdk index 9bffb525..582e0241 160000 --- a/test-server/ruby-v3-server/local-ruby-sdk +++ b/test-server/ruby-v3-server/local-ruby-sdk @@ -1 +1 @@ -Subproject commit 9bffb525765b2aeebd6203072bf6e94d2ab53f90 +Subproject commit 582e02418ac2c5540c48dd089a7db506712e6f94 diff --git a/test-server/specification b/test-server/specification index 129b9c5e..1f1ae8bb 160000 --- a/test-server/specification +++ b/test-server/specification @@ -1 +1 @@ -Subproject commit 129b9c5e53a8c4f6be10a52c9d3dcdf765000d78 +Subproject commit 1f1ae8bb2b7b082b87ffbf4916a9723e531b2052 From d21fd62200cd7a80665cc7c4283a45b8ac28ed83 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Mon, 10 Nov 2025 16:15:32 -0800 Subject: [PATCH 02/45] Go --- test-server/go-v3-transition-server/local-go-s3ec | 2 +- test-server/go-v4-server/local-go-s3ec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test-server/go-v3-transition-server/local-go-s3ec b/test-server/go-v3-transition-server/local-go-s3ec index 7a29344c..f6de57fd 160000 --- a/test-server/go-v3-transition-server/local-go-s3ec +++ b/test-server/go-v3-transition-server/local-go-s3ec @@ -1 +1 @@ -Subproject commit 7a29344cc0c431fd5ac6d0a08ce4db455d75c175 +Subproject commit f6de57fda0a2fc5fa689e64f1569ee08b6dcfa8d diff --git a/test-server/go-v4-server/local-go-s3ec b/test-server/go-v4-server/local-go-s3ec index 9946186d..f6de57fd 160000 --- a/test-server/go-v4-server/local-go-s3ec +++ b/test-server/go-v4-server/local-go-s3ec @@ -1 +1 @@ -Subproject commit 9946186d6b760074750a535b663d6c84c5815308 +Subproject commit f6de57fda0a2fc5fa689e64f1569ee08b6dcfa8d From 1a3e1e15f000c02510a026144aba6fc59272ba3c Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 08:57:16 -0800 Subject: [PATCH 03/45] Adding Cpp to submodules --- .github/workflows/test.yml | 34 +++++++++---------- .gitmodules | 8 +++++ .../cpp-v2-transition-server/aws-sdk-cpp | 1 + test-server/cpp-v3-server/aws-sdk-cpp | 1 + 4 files changed, 27 insertions(+), 17 deletions(-) create mode 160000 test-server/cpp-v2-transition-server/aws-sdk-cpp create mode 160000 test-server/cpp-v3-server/aws-sdk-cpp diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c29cfb5a..f21b45e5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,23 +27,23 @@ jobs: # ask Ryan to edit this PAT's permissions to add access to a new private repo. token: ${{ secrets.PAT_FOR_PRIVATE_RUBY }} - - name: Checkout CPP code for cpp-v2-transition - uses: actions/checkout@v5 - with: - submodules: recursive - token: ${{ secrets.PAT_FOR_CPP }} - repository: awslabs/aws-sdk-cpp-staging - ref: fire-egg-dev - path: test-server/cpp-v2-transition-server/aws-sdk-cpp/ - - - name: Checkout CPP code cpp-v3 - uses: actions/checkout@v5 - with: - submodules: recursive - token: ${{ secrets.PAT_FOR_CPP }} - repository: awslabs/aws-sdk-cpp-staging - ref: fire-egg-dev - path: test-server/cpp-v3-server/aws-sdk-cpp/ + # - name: Checkout CPP code for cpp-v2-transition + # uses: actions/checkout@v5 + # with: + # submodules: recursive + # token: ${{ secrets.PAT_FOR_CPP }} + # repository: awslabs/aws-sdk-cpp-staging + # ref: fire-egg-dev + # path: test-server/cpp-v2-transition-server/aws-sdk-cpp/ + + # - name: Checkout CPP code cpp-v3 + # uses: actions/checkout@v5 + # with: + # submodules: recursive + # token: ${{ secrets.PAT_FOR_CPP }} + # repository: awslabs/aws-sdk-cpp-staging + # ref: fire-egg-dev + # path: test-server/cpp-v3-server/aws-sdk-cpp/ - name: Checkout .NET V2 code uses: actions/checkout@v5 diff --git a/.gitmodules b/.gitmodules index e3080fcf..ca9714e8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -52,3 +52,11 @@ path = test-server/net-v3-transition-server/s3ec-v3-transition-branch url = https://github.com/aws/private-amazon-s3-encryption-client-dotnet-staging.git branch = rishav/key-commitment +[submodule "test-server/cpp-v2-transition-server/aws-sdk-cpp"] + path = test-server/cpp-v2-transition-server/aws-sdk-cpp + url = git@github.com:awslabs/aws-sdk-cpp-staging.git + branch = fire-egg-dev +[submodule "test-server/cpp-v3-server/aws-sdk-cpp"] + path = test-server/cpp-v3-server/aws-sdk-cpp + url = git@github.com:awslabs/aws-sdk-cpp-staging.git + branch = fire-egg-dev diff --git a/test-server/cpp-v2-transition-server/aws-sdk-cpp b/test-server/cpp-v2-transition-server/aws-sdk-cpp new file mode 160000 index 00000000..7f74e710 --- /dev/null +++ b/test-server/cpp-v2-transition-server/aws-sdk-cpp @@ -0,0 +1 @@ +Subproject commit 7f74e710cc52ef08cbd9bafaa79904b415efb988 diff --git a/test-server/cpp-v3-server/aws-sdk-cpp b/test-server/cpp-v3-server/aws-sdk-cpp new file mode 160000 index 00000000..7f74e710 --- /dev/null +++ b/test-server/cpp-v3-server/aws-sdk-cpp @@ -0,0 +1 @@ +Subproject commit 7f74e710cc52ef08cbd9bafaa79904b415efb988 From 761dda7bc348d140ef4445a5551ad5d606bb2f4d Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 09:09:13 -0800 Subject: [PATCH 04/45] recursive --- .github/workflows/test.yml | 50 +++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f21b45e5..4790d2f9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ jobs: - name: Checkout code uses: actions/checkout@v5 with: - submodules: true + submodules: recursive # This is Ryan Emery's (seebees) PAT. # To grant this workflow access to a new private repo, # ask Ryan to edit this PAT's permissions to add access to a new private repo. @@ -45,32 +45,32 @@ jobs: # ref: fire-egg-dev # path: test-server/cpp-v3-server/aws-sdk-cpp/ - - name: Checkout .NET V2 code - uses: actions/checkout@v5 - with: - token: ${{ secrets.PAT_FOR_DOTNET }} - repository: aws/private-amazon-s3-encryption-client-dotnet-staging - # This is the branch for S3EC .NET V2 - ref: v3sdk-development - path: test-server/net-v2-v3-server/s3ec-net-v2/ + # - name: Checkout .NET V2 code + # uses: actions/checkout@v5 + # with: + # token: ${{ secrets.PAT_FOR_DOTNET }} + # repository: aws/private-amazon-s3-encryption-client-dotnet-staging + # # This is the branch for S3EC .NET V2 + # ref: v3sdk-development + # path: test-server/net-v2-v3-server/s3ec-net-v2/ - - name: Checkout .NET V3 code - uses: actions/checkout@v5 - with: - token: ${{ secrets.PAT_FOR_DOTNET }} - repository: aws/private-amazon-s3-encryption-client-dotnet-staging - # This is the branch for S3EC .NET V3 - ref: s3ec-v3 - path: test-server/net-v2-v3-server/s3ec-net-v3 + # - name: Checkout .NET V3 code + # uses: actions/checkout@v5 + # with: + # token: ${{ secrets.PAT_FOR_DOTNET }} + # repository: aws/private-amazon-s3-encryption-client-dotnet-staging + # # This is the branch for S3EC .NET V3 + # ref: s3ec-v3 + # path: test-server/net-v2-v3-server/s3ec-net-v3 - - name: Checkout .NET V3 code (Transition) - uses: actions/checkout@v5 - with: - token: ${{ secrets.PAT_FOR_DOTNET }} - repository: aws/private-amazon-s3-encryption-client-dotnet-staging - # This is the branch for S3EC .NET V3 transition - ref: rishav/key-commitment - path: test-server/net-v3-transition-server/s3ec-v3-transition-branch + # - name: Checkout .NET V3 code (Transition) + # uses: actions/checkout@v5 + # with: + # token: ${{ secrets.PAT_FOR_DOTNET }} + # repository: aws/private-amazon-s3-encryption-client-dotnet-staging + # # This is the branch for S3EC .NET V3 transition + # ref: rishav/key-commitment + # path: test-server/net-v3-transition-server/s3ec-v3-transition-branch - name: Set up Python uses: actions/setup-python@v5 From 7e6d1f27d2f9c2a8444ae4109a233c5564946d98 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 09:18:35 -0800 Subject: [PATCH 05/45] Add cpp main to submodule Also update tests to only do interopt --- .github/workflows/main.yml | 2 +- .github/workflows/test.yml | 86 +++++++-------------------- .gitmodules | 3 + test-server/cpp-v2-server/Makefile | 1 - test-server/cpp-v2-server/aws-sdk-cpp | 1 + 5 files changed, 26 insertions(+), 67 deletions(-) create mode 160000 test-server/cpp-v2-server/aws-sdk-cpp diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d53b9dca..9acc9e7a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,7 +21,7 @@ jobs: permissions: id-token: write contents: read - name: Run Tests + name: Run Interoperability Tests uses: ./.github/workflows/test.yml with: python-version: ${{ inputs.python-version || '3.11' }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4790d2f9..90ca4893 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,51 +27,6 @@ jobs: # ask Ryan to edit this PAT's permissions to add access to a new private repo. token: ${{ secrets.PAT_FOR_PRIVATE_RUBY }} - # - name: Checkout CPP code for cpp-v2-transition - # uses: actions/checkout@v5 - # with: - # submodules: recursive - # token: ${{ secrets.PAT_FOR_CPP }} - # repository: awslabs/aws-sdk-cpp-staging - # ref: fire-egg-dev - # path: test-server/cpp-v2-transition-server/aws-sdk-cpp/ - - # - name: Checkout CPP code cpp-v3 - # uses: actions/checkout@v5 - # with: - # submodules: recursive - # token: ${{ secrets.PAT_FOR_CPP }} - # repository: awslabs/aws-sdk-cpp-staging - # ref: fire-egg-dev - # path: test-server/cpp-v3-server/aws-sdk-cpp/ - - # - name: Checkout .NET V2 code - # uses: actions/checkout@v5 - # with: - # token: ${{ secrets.PAT_FOR_DOTNET }} - # repository: aws/private-amazon-s3-encryption-client-dotnet-staging - # # This is the branch for S3EC .NET V2 - # ref: v3sdk-development - # path: test-server/net-v2-v3-server/s3ec-net-v2/ - - # - name: Checkout .NET V3 code - # uses: actions/checkout@v5 - # with: - # token: ${{ secrets.PAT_FOR_DOTNET }} - # repository: aws/private-amazon-s3-encryption-client-dotnet-staging - # # This is the branch for S3EC .NET V3 - # ref: s3ec-v3 - # path: test-server/net-v2-v3-server/s3ec-net-v3 - - # - name: Checkout .NET V3 code (Transition) - # uses: actions/checkout@v5 - # with: - # token: ${{ secrets.PAT_FOR_DOTNET }} - # repository: aws/private-amazon-s3-encryption-client-dotnet-staging - # # This is the branch for S3EC .NET V3 transition - # ref: rishav/key-commitment - # path: test-server/net-v3-transition-server/s3ec-v3-transition-branch - - name: Set up Python uses: actions/setup-python@v5 with: @@ -107,17 +62,27 @@ jobs: with: go-version: 1.25 - # Cache uv dependencies - - name: Cache uv dependencies - uses: actions/cache@v3 - with: - path: ~/.cache/uv - key: ${{ runner.os }}-uv-${{ hashFiles('./test-server/python-v3-server/**/pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-uv- +# Legacy Python tests: + # # Cache uv dependencies + # - name: Cache uv dependencies + # uses: actions/cache@v3 + # with: + # path: ~/.cache/uv + # key: ${{ runner.os }}-uv-${{ hashFiles('./test-server/python-v3-server/**/pyproject.toml') }} + # restore-keys: | + # ${{ runner.os }}-uv- + + # - name: Install Uv + # run: pip install uv + + # - name: Run unit tests + # run: make test-unit - - name: Install Uv - run: pip install uv + # - name: Run integration tests + # run: make test-integration + # env: + # CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} + # CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} # Cache Gradle dependencies and build outputs - name: Cache Gradle packages @@ -141,17 +106,8 @@ jobs: role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role aws-region: us-west-2 - - name: Run unit tests - run: make test-unit - - - name: Run integration tests - run: make test-integration - env: - CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }} - CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }} - - name: Run test-server tests - run: cd test-server && make ci + run: cd test-server && make ci FILTER=ruby,go env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} diff --git a/.gitmodules b/.gitmodules index ca9714e8..4b17968f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -60,3 +60,6 @@ path = test-server/cpp-v3-server/aws-sdk-cpp url = git@github.com:awslabs/aws-sdk-cpp-staging.git branch = fire-egg-dev +[submodule "test-server/cpp-v2-server/aws-sdk-cpp"] + path = test-server/cpp-v2-server/aws-sdk-cpp + url = git@github.com:awslabs/aws-sdk-cpp-staging.git diff --git a/test-server/cpp-v2-server/Makefile b/test-server/cpp-v2-server/Makefile index 9e0f04b1..231f6d5b 100644 --- a/test-server/cpp-v2-server/Makefile +++ b/test-server/cpp-v2-server/Makefile @@ -7,7 +7,6 @@ PORT := 8085 build/s3ec-server: brew install libmicrohttpd nlohmann-json ossp-uuid - git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git cd aws-sdk-cpp mkdir -p build && cd build && cmake .. diff --git a/test-server/cpp-v2-server/aws-sdk-cpp b/test-server/cpp-v2-server/aws-sdk-cpp new file mode 160000 index 00000000..994384ca --- /dev/null +++ b/test-server/cpp-v2-server/aws-sdk-cpp @@ -0,0 +1 @@ +Subproject commit 994384ca8b9defe2ae60b5d3447ec5f47f7ec19f From f070451dfb5c1de46ab8cf96286bcc77f913cc1c Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 10:02:24 -0800 Subject: [PATCH 06/45] Adding cpp and some updates --- .github/workflows/main.yml | 2 +- .github/workflows/test.yml | 12 ++++++------ test-server/cpp-v2-transition-server/aws-sdk-cpp | 2 +- test-server/cpp-v3-server/aws-sdk-cpp | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9acc9e7a..d53b9dca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,7 +21,7 @@ jobs: permissions: id-token: write contents: read - name: Run Interoperability Tests + name: Run Tests uses: ./.github/workflows/test.yml with: python-version: ${{ inputs.python-version || '3.11' }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 90ca4893..71bea59b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,11 +27,6 @@ jobs: # ask Ryan to edit this PAT's permissions to add access to a new private repo. token: ${{ secrets.PAT_FOR_PRIVATE_RUBY }} - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python-version || '3.11' }} - - name: Set up Ruby uses: ruby/setup-ruby@v1 with: @@ -62,7 +57,12 @@ jobs: with: go-version: 1.25 -# Legacy Python tests: + # Legacy Python tests: + # - name: Set up Python + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ inputs.python-version || '3.11' }} + # # # Cache uv dependencies # - name: Cache uv dependencies # uses: actions/cache@v3 diff --git a/test-server/cpp-v2-transition-server/aws-sdk-cpp b/test-server/cpp-v2-transition-server/aws-sdk-cpp index 7f74e710..9a368aa8 160000 --- a/test-server/cpp-v2-transition-server/aws-sdk-cpp +++ b/test-server/cpp-v2-transition-server/aws-sdk-cpp @@ -1 +1 @@ -Subproject commit 7f74e710cc52ef08cbd9bafaa79904b415efb988 +Subproject commit 9a368aa8f6bbcd75eb1180d5d76d41936c68ed6a diff --git a/test-server/cpp-v3-server/aws-sdk-cpp b/test-server/cpp-v3-server/aws-sdk-cpp index 7f74e710..9a368aa8 160000 --- a/test-server/cpp-v3-server/aws-sdk-cpp +++ b/test-server/cpp-v3-server/aws-sdk-cpp @@ -1 +1 @@ -Subproject commit 7f74e710cc52ef08cbd9bafaa79904b415efb988 +Subproject commit 9a368aa8f6bbcd75eb1180d5d76d41936c68ed6a From a8e4267fb9576e037d5ed8426cf6b8ae080b4204 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 10:14:21 -0800 Subject: [PATCH 07/45] simplify submodules --- .gitmodules | 4 ---- .../java-v3-transition-server/.duvet/config.toml | 14 +++++++------- .../java-v3-transition-server/specification | 1 - test-server/java-v4-server/.duvet/config.toml | 14 +++++++------- test-server/java-v4-server/specification | 1 - 5 files changed, 14 insertions(+), 20 deletions(-) delete mode 120000 test-server/java-v3-transition-server/specification delete mode 120000 test-server/java-v4-server/specification diff --git a/.gitmodules b/.gitmodules index 4b17968f..368741d8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -24,10 +24,6 @@ url = git@github.com:aws/private-amazon-s3-encryption-client-java-staging.git branch = imabhichow/add-kc ; branch = s3ec/improved -[submodule "test-server/java-v4-server/specification"] - path = test-server/java-v4-server/specification - url = git@github.com:awslabs/private-aws-encryption-sdk-specification-staging.git - branch = fire-egg-staging [submodule "test-server/specification"] path = test-server/specification url = git@github.com:awslabs/private-aws-encryption-sdk-specification-staging.git diff --git a/test-server/java-v3-transition-server/.duvet/config.toml b/test-server/java-v3-transition-server/.duvet/config.toml index d07014da..f29cd058 100644 --- a/test-server/java-v3-transition-server/.duvet/config.toml +++ b/test-server/java-v3-transition-server/.duvet/config.toml @@ -5,19 +5,19 @@ pattern = "s3ec-staging/*.java" # Include required specifications here [[specification]] -source = "specification/s3-encryption/client.md" +source = "../specification/s3-encryption/client.md" [[specification]] -source = "specification/s3-encryption/decryption.md" +source = "../specification/s3-encryption/decryption.md" [[specification]] -source = "specification/s3-encryption/encryption.md" +source = "../specification/s3-encryption/encryption.md" [[specification]] -source = "specification/s3-encryption/key-commitment.md" +source = "../specification/s3-encryption/key-commitment.md" [[specification]] -source = "specification/s3-encryption/key-derivation.md" +source = "../specification/s3-encryption/key-derivation.md" [[specification]] -source = "specification/s3-encryption/data-format/content-metadata.md" +source = "../specification/s3-encryption/data-format/content-metadata.md" [[specification]] -source = "specification/s3-encryption/data-format/metadata-strategy.md" +source = "../specification/s3-encryption/data-format/metadata-strategy.md" [report.html] enabled = true diff --git a/test-server/java-v3-transition-server/specification b/test-server/java-v3-transition-server/specification deleted file mode 120000 index b173f708..00000000 --- a/test-server/java-v3-transition-server/specification +++ /dev/null @@ -1 +0,0 @@ -../specification \ No newline at end of file diff --git a/test-server/java-v4-server/.duvet/config.toml b/test-server/java-v4-server/.duvet/config.toml index d07014da..f29cd058 100644 --- a/test-server/java-v4-server/.duvet/config.toml +++ b/test-server/java-v4-server/.duvet/config.toml @@ -5,19 +5,19 @@ pattern = "s3ec-staging/*.java" # Include required specifications here [[specification]] -source = "specification/s3-encryption/client.md" +source = "../specification/s3-encryption/client.md" [[specification]] -source = "specification/s3-encryption/decryption.md" +source = "../specification/s3-encryption/decryption.md" [[specification]] -source = "specification/s3-encryption/encryption.md" +source = "../specification/s3-encryption/encryption.md" [[specification]] -source = "specification/s3-encryption/key-commitment.md" +source = "../specification/s3-encryption/key-commitment.md" [[specification]] -source = "specification/s3-encryption/key-derivation.md" +source = "../specification/s3-encryption/key-derivation.md" [[specification]] -source = "specification/s3-encryption/data-format/content-metadata.md" +source = "../specification/s3-encryption/data-format/content-metadata.md" [[specification]] -source = "specification/s3-encryption/data-format/metadata-strategy.md" +source = "../specification/s3-encryption/data-format/metadata-strategy.md" [report.html] enabled = true diff --git a/test-server/java-v4-server/specification b/test-server/java-v4-server/specification deleted file mode 120000 index b173f708..00000000 --- a/test-server/java-v4-server/specification +++ /dev/null @@ -1 +0,0 @@ -../specification \ No newline at end of file From 97b9eabaf5a07ce370e2993d90070318e5f28713 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 10:45:02 -0800 Subject: [PATCH 08/45] only cpp recursive --- .github/workflows/test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 71bea59b..1b66f606 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,12 +21,17 @@ jobs: - name: Checkout code uses: actions/checkout@v5 with: - submodules: recursive + submodules: true # This is Ryan Emery's (seebees) PAT. # To grant this workflow access to a new private repo, # ask Ryan to edit this PAT's permissions to add access to a new private repo. token: ${{ secrets.PAT_FOR_PRIVATE_RUBY }} + - name: Update cpp submodules recursively + run: | + git submodule update --init --recursive --force test-server/cpp-v2-transition-server/aws-sdk-cpp + git submodule update --init --recursive --force test-server/cpp-v3-server/aws-sdk-cpp + git submodule update --init --recursive --force test-server/cpp-v2-server/aws-sdk-cpp - name: Set up Ruby uses: ruby/setup-ruby@v1 with: From 9674410109f63abe8daeee3743c02837da65df86 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 10:59:27 -0800 Subject: [PATCH 09/45] More python :( --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1b66f606..d281f151 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -80,6 +80,9 @@ jobs: # - name: Install Uv # run: pip install uv + # - name: Install dependencies + # run: make install + # - name: Run unit tests # run: make test-unit @@ -102,9 +105,6 @@ jobs: restore-keys: | ${{ runner.os }}-gradle- - - name: Install dependencies - run: make install - - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: From 13f2d2722b20d6f22bd6c127f34c21bfc989dc13 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 11:17:47 -0800 Subject: [PATCH 10/45] update go --- .github/workflows/test.yml | 1 + test-server/go-v3-transition-server/local-go-s3ec | 2 +- test-server/go-v4-server/local-go-s3ec | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d281f151..ab2249b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,6 +32,7 @@ jobs: git submodule update --init --recursive --force test-server/cpp-v2-transition-server/aws-sdk-cpp git submodule update --init --recursive --force test-server/cpp-v3-server/aws-sdk-cpp git submodule update --init --recursive --force test-server/cpp-v2-server/aws-sdk-cpp + - name: Set up Ruby uses: ruby/setup-ruby@v1 with: diff --git a/test-server/go-v3-transition-server/local-go-s3ec b/test-server/go-v3-transition-server/local-go-s3ec index f6de57fd..c295293d 160000 --- a/test-server/go-v3-transition-server/local-go-s3ec +++ b/test-server/go-v3-transition-server/local-go-s3ec @@ -1 +1 @@ -Subproject commit f6de57fda0a2fc5fa689e64f1569ee08b6dcfa8d +Subproject commit c295293d99449798be26e31bb506016a9141fa16 diff --git a/test-server/go-v4-server/local-go-s3ec b/test-server/go-v4-server/local-go-s3ec index f6de57fd..c295293d 160000 --- a/test-server/go-v4-server/local-go-s3ec +++ b/test-server/go-v4-server/local-go-s3ec @@ -1 +1 @@ -Subproject commit f6de57fda0a2fc5fa689e64f1569ee08b6dcfa8d +Subproject commit c295293d99449798be26e31bb506016a9141fa16 From d2ea5e43295dbd3eee68d824bfcb1ea1197be843 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 11:47:30 -0800 Subject: [PATCH 11/45] updates to the other one. --- test-server/go-v3-transition-server/local-go-s3ec | 2 +- test-server/go-v4-server/local-go-s3ec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test-server/go-v3-transition-server/local-go-s3ec b/test-server/go-v3-transition-server/local-go-s3ec index c295293d..85fd30c6 160000 --- a/test-server/go-v3-transition-server/local-go-s3ec +++ b/test-server/go-v3-transition-server/local-go-s3ec @@ -1 +1 @@ -Subproject commit c295293d99449798be26e31bb506016a9141fa16 +Subproject commit 85fd30c6a7ebbef3d056991c6f3673e0e9002bcf diff --git a/test-server/go-v4-server/local-go-s3ec b/test-server/go-v4-server/local-go-s3ec index c295293d..85fd30c6 160000 --- a/test-server/go-v4-server/local-go-s3ec +++ b/test-server/go-v4-server/local-go-s3ec @@ -1 +1 @@ -Subproject commit c295293d99449798be26e31bb506016a9141fa16 +Subproject commit 85fd30c6a7ebbef3d056991c6f3673e0e9002bcf From e406188b04dd362986bf0c2392aa87be57ec0cfb Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 12:12:48 -0800 Subject: [PATCH 12/45] faster CPP init --- .github/workflows/test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ab2249b4..e825f65a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,9 +29,7 @@ jobs: - name: Update cpp submodules recursively run: | - git submodule update --init --recursive --force test-server/cpp-v2-transition-server/aws-sdk-cpp - git submodule update --init --recursive --force test-server/cpp-v3-server/aws-sdk-cpp - git submodule update --init --recursive --force test-server/cpp-v2-server/aws-sdk-cpp + git submodule update --init --recursive --jobs 4 --force test-server/cpp-v2-transition-server/aws-sdk-cpp test-server/cpp-v3-server/aws-sdk-cpp test-server/cpp-v2-server/aws-sdk-cpp - name: Set up Ruby uses: ruby/setup-ruby@v1 From fb0c369238516d17615b19ad70d5412e25e9cbf2 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 12:26:20 -0800 Subject: [PATCH 13/45] maybe faster network --- .github/workflows/test.yml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e825f65a..f436a4cc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,15 +21,33 @@ jobs: - name: Checkout code uses: actions/checkout@v5 with: - submodules: true + submodules: false # This is Ryan Emery's (seebees) PAT. # To grant this workflow access to a new private repo, # ask Ryan to edit this PAT's permissions to add access to a new private repo. token: ${{ secrets.PAT_FOR_PRIVATE_RUBY }} + # There are a lot of submodules here + # This initializes the checkouts in parallel (--jobs) + # rather than in series the way actions/checkout@v5 does it. + + - name: Get CPU count + id: cpu-count + run: echo "count=$(node -p 'require("os").cpus().length')" >> $GITHUB_OUTPUT + + - name: Checkout submodules with PAT + run: | + git config --global url."https://x-token-auth:${{ secrets.PAT_FOR_PRIVATE_RUBY }}@github.com/".insteadOf "https://github.com/" + git submodule update --init --jobs ${{ steps.cpu-count.outputs.count }} + - name: Update cpp submodules recursively run: | - git submodule update --init --recursive --jobs 4 --force test-server/cpp-v2-transition-server/aws-sdk-cpp test-server/cpp-v3-server/aws-sdk-cpp test-server/cpp-v2-server/aws-sdk-cpp + git submodule update --init --recursive \ + --jobs ${{ steps.cpu-count.outputs.count }} \ + --force \ + test-server/cpp-v2-transition-server/aws-sdk-cpp \ + test-server/cpp-v3-server/aws-sdk-cpp \ + test-server/cpp-v2-server/aws-sdk-cpp - name: Set up Ruby uses: ruby/setup-ruby@v1 From edb12e1eed671a7b6dd156bc9cbf53b88cb19c92 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 12:30:52 -0800 Subject: [PATCH 14/45] add short? --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f436a4cc..2800b0d8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,6 +38,7 @@ jobs: - name: Checkout submodules with PAT run: | git config --global url."https://x-token-auth:${{ secrets.PAT_FOR_PRIVATE_RUBY }}@github.com/".insteadOf "https://github.com/" + git config --global url."https://x-token-auth:${{ secrets.PAT_FOR_PRIVATE_RUBY }}@github.com/".insteadOf "git@github.com:" git submodule update --init --jobs ${{ steps.cpu-count.outputs.count }} - name: Update cpp submodules recursively From 78cbe369f337638c9e870206b83536cc0e0082a7 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 12:36:38 -0800 Subject: [PATCH 15/45] getting out of hand --- .github/workflows/test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2800b0d8..aa98c821 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,10 +37,12 @@ jobs: - name: Checkout submodules with PAT run: | - git config --global url."https://x-token-auth:${{ secrets.PAT_FOR_PRIVATE_RUBY }}@github.com/".insteadOf "https://github.com/" - git config --global url."https://x-token-auth:${{ secrets.PAT_FOR_PRIVATE_RUBY }}@github.com/".insteadOf "git@github.com:" + git config --global url."https://github.com/".insteadOf "git@github.com:" + git config --global credential.helper store + echo "https://x-token-auth:${{ secrets.PAT_FOR_PRIVATE_RUBY }}@github.com" > ~/.git-credentials git submodule update --init --jobs ${{ steps.cpu-count.outputs.count }} + - name: Update cpp submodules recursively run: | git submodule update --init --recursive \ From b0fbd048c309852644c433f394d133bfa9b3a2ec Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 12:43:08 -0800 Subject: [PATCH 16/45] makeing jobs --- test-server/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test-server/Makefile b/test-server/Makefile index dc3bdad3..34d3414d 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -1,5 +1,7 @@ # Makefile for S3 Encryption Client Testing +export MAKEFLAGS=-j$(shell nproc) + .PHONY: all start-servers run-tests stop-servers clean ci check-env help # Default target From 33a96ab1fda19f3e4ef44945cffb2a492c4a74b6 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 12:53:10 -0800 Subject: [PATCH 17/45] try running brew only once --- .github/workflows/test.yml | 5 ++++- test-server/cpp-v2-server/Makefile | 3 +-- test-server/cpp-v2-transition-server/Makefile | 1 - test-server/cpp-v3-server/Makefile | 1 - 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aa98c821..f5a987f1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,6 @@ jobs: echo "https://x-token-auth:${{ secrets.PAT_FOR_PRIVATE_RUBY }}@github.com" > ~/.git-credentials git submodule update --init --jobs ${{ steps.cpu-count.outputs.count }} - - name: Update cpp submodules recursively run: | git submodule update --init --recursive \ @@ -82,6 +81,10 @@ jobs: with: go-version: 1.25 + - name: Install C++ dependencies + run: | + brew install libmicrohttpd nlohmann-json ossp-uuid + # Legacy Python tests: # - name: Set up Python # uses: actions/setup-python@v5 diff --git a/test-server/cpp-v2-server/Makefile b/test-server/cpp-v2-server/Makefile index 231f6d5b..ed229e86 100644 --- a/test-server/cpp-v2-server/Makefile +++ b/test-server/cpp-v2-server/Makefile @@ -5,8 +5,7 @@ PID_FILE := server.pid PORT := 8085 -build/s3ec-server: - brew install libmicrohttpd nlohmann-json ossp-uuid +build/s3ec-server: cd aws-sdk-cpp mkdir -p build && cd build && cmake .. diff --git a/test-server/cpp-v2-transition-server/Makefile b/test-server/cpp-v2-transition-server/Makefile index 05803c78..cd725209 100644 --- a/test-server/cpp-v2-transition-server/Makefile +++ b/test-server/cpp-v2-transition-server/Makefile @@ -6,7 +6,6 @@ PID_FILE := server.pid PORT := 8097 build/s3ec-server: - brew install libmicrohttpd nlohmann-json ossp-uuid mkdir -p build && cd build && cmake .. start-server: | build/s3ec-server diff --git a/test-server/cpp-v3-server/Makefile b/test-server/cpp-v3-server/Makefile index 86fc285e..6be22b87 100644 --- a/test-server/cpp-v3-server/Makefile +++ b/test-server/cpp-v3-server/Makefile @@ -6,7 +6,6 @@ PID_FILE := server.pid PORT := 8091 build/s3ec-server: - brew install libmicrohttpd nlohmann-json ossp-uuid mkdir -p build && cd build && cmake .. start-server: | build/s3ec-server From c8d94d22f86fe32d90b430ff79e7ff42aa1aec71 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 12:54:51 -0800 Subject: [PATCH 18/45] Add c++ --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f5a987f1..fe6494c3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -135,7 +135,7 @@ jobs: aws-region: us-west-2 - name: Run test-server tests - run: cd test-server && make ci FILTER=ruby,go + run: cd test-server && make ci FILTER=ruby,go,cpp env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} From 81f7cae63bc260e914b2ba0effc669e7b5e9b78f Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 13:20:36 -0800 Subject: [PATCH 19/45] better logging for C --- test-server/cpp-v2-server/Makefile | 3 ++- test-server/cpp-v2-transition-server/Makefile | 5 +++-- test-server/cpp-v3-server/Makefile | 5 +++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/test-server/cpp-v2-server/Makefile b/test-server/cpp-v2-server/Makefile index ed229e86..f060d569 100644 --- a/test-server/cpp-v2-server/Makefile +++ b/test-server/cpp-v2-server/Makefile @@ -10,8 +10,9 @@ build/s3ec-server: mkdir -p build && cd build && cmake .. start-server: | build/s3ec-server - @echo "Starting Cpp V2 server..." + @echo "Building Cpp V2 server..." cd build && make && \ + @echo "Starting Cpp V2 server..." AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ diff --git a/test-server/cpp-v2-transition-server/Makefile b/test-server/cpp-v2-transition-server/Makefile index cd725209..82683cf4 100644 --- a/test-server/cpp-v2-transition-server/Makefile +++ b/test-server/cpp-v2-transition-server/Makefile @@ -9,14 +9,15 @@ build/s3ec-server: mkdir -p build && cd build && cmake .. start-server: | build/s3ec-server - @echo "Starting Cpp V2 server..." + @echo "Building Cpp transition server..." cd build && make && \ + @echo "Starting Cpp transition server..." AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ ./s3ec-server & echo $$! > $(PID_FILE) - @echo "Cpp V2 server starting..." + @echo "Cpp transition server starting..." stop-server: @if [ -f $(PID_FILE) ]; then \ diff --git a/test-server/cpp-v3-server/Makefile b/test-server/cpp-v3-server/Makefile index 6be22b87..537eb402 100644 --- a/test-server/cpp-v3-server/Makefile +++ b/test-server/cpp-v3-server/Makefile @@ -9,14 +9,15 @@ build/s3ec-server: mkdir -p build && cd build && cmake .. start-server: | build/s3ec-server - @echo "Starting Cpp V2 server..." + @echo "Building Cpp V3 server..." cd build && make && \ + @echo "Starting Cpp V3 server..." AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ ./s3ec-server & echo $$! > $(PID_FILE) - @echo "Cpp V2 server starting..." + @echo "Cpp V3 server starting..." stop-server: @if [ -f $(PID_FILE) ]; then \ From 9348dfb56885cd250f325f5e6ad3554775727cd5 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 13:25:01 -0800 Subject: [PATCH 20/45] depth 1 --- .github/workflows/test.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fe6494c3..2963c2d8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,16 +35,19 @@ jobs: id: cpu-count run: echo "count=$(node -p 'require("os").cpus().length')" >> $GITHUB_OUTPUT - - name: Checkout submodules with PAT + - name: Setup git submodules with PAT run: | git config --global url."https://github.com/".insteadOf "git@github.com:" git config --global credential.helper store echo "https://x-token-auth:${{ secrets.PAT_FOR_PRIVATE_RUBY }}@github.com" > ~/.git-credentials - git submodule update --init --jobs ${{ steps.cpu-count.outputs.count }} + + - name: Checkout submodules with jobs + run: | + git submodule update --init --depth 1 --jobs ${{ steps.cpu-count.outputs.count }} - name: Update cpp submodules recursively run: | - git submodule update --init --recursive \ + git submodule update --init --depth 1 --recursive \ --jobs ${{ steps.cpu-count.outputs.count }} \ --force \ test-server/cpp-v2-transition-server/aws-sdk-cpp \ From 30b435aed7b18654a905a97a3ea7d8281e9defc6 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 13:42:52 -0800 Subject: [PATCH 21/45] Port and make -j --- test-server/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test-server/Makefile b/test-server/Makefile index 34d3414d..4edc8984 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -1,6 +1,6 @@ # Makefile for S3 Encryption Client Testing -export MAKEFLAGS=-j$(shell nproc) +export MAKEFLAGS=-j$(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 1) .PHONY: all start-servers run-tests stop-servers clean ci check-env help @@ -96,6 +96,7 @@ wait-for-port: echo "❌ Error: PORT is required"; \ exit 1; \ fi + echo "Starting to wait for $$PORT to start"; \ @for i in $$(seq 1 $(TIMEOUT)); do \ if nc -z localhost $$PORT; then \ echo "Ports are open, waiting for servers to initialize..."; \ From 9037988df417080e0e1b7d9ef6e4b41be6d86818 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 13:48:17 -0800 Subject: [PATCH 22/45] CI should be faster --- test-server/Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test-server/Makefile b/test-server/Makefile index 4edc8984..4a8e45ca 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -8,7 +8,7 @@ export MAKEFLAGS=-j$(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || all: start-all-servers wait-all-servers run-tests # CI target for GitHub Actions -ci: start-servers run-tests stop-servers +ci: start-all-servers wait-all-servers run-tests stop-servers SERVER_DIRS := $(shell find . -maxdepth 1 -type d -name '*-server' | sed 's|^\./||' | $(if $(FILTER),grep -E "$$(echo '$(FILTER)' | sed 's/,/|/g')",cat) | sort) @@ -36,7 +36,10 @@ $(START_SERVER_TARGETS): start-%: exit 1; \ fi; \ -wait-all-servers: $(WAIT_SERVER_TARGETS) +wait-all-servers: + @echo "Waiting for all servers to be ready..." + $(MAKE) $(WAIT_SERVER_TARGETS) + @echo "All servers are ready!" $(WAIT_SERVER_TARGETS): wait-%: @if [ -f $*/Makefile ]; then \ From bd2e0982659e9b81d8e09aaabf36c573f8f9b9f2 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 13:59:41 -0800 Subject: [PATCH 23/45] checkout faster, now with cache! --- .github/workflows/test.yml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2963c2d8..4c36ed32 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,15 +41,31 @@ jobs: git config --global credential.helper store echo "https://x-token-auth:${{ secrets.PAT_FOR_PRIVATE_RUBY }}@github.com" > ~/.git-credentials - - name: Checkout submodules with jobs + - name: Cache git submodules + uses: actions/cache@v4 + with: + path: | + .git/modules + test-server/*/.git + key: ${{ runner.os }}-submodules-${{ hashFiles('.gitmodules') }} + restore-keys: | + ${{ runner.os }}-submodules- + + - name: Optimize git for performance run: | - git submodule update --init --depth 1 --jobs ${{ steps.cpu-count.outputs.count }} + git config --global fetch.parallel ${{ steps.cpu-count.outputs.count }} + git config --global submodule.fetchJobs ${{ steps.cpu-count.outputs.count }} - - name: Update cpp submodules recursively + - name: Checkout all submodules (use --jobs) run: | - git submodule update --init --depth 1 --recursive \ + # Non-recursive for most submodules + git submodule update --init --depth 1 --jobs ${{ steps.cpu-count.outputs.count }} + + # Recursive only for C++ SDK submodules with blobless clone + git submodule update --init --recursive \ + --depth 1 \ + --filter=blob:none \ --jobs ${{ steps.cpu-count.outputs.count }} \ - --force \ test-server/cpp-v2-transition-server/aws-sdk-cpp \ test-server/cpp-v3-server/aws-sdk-cpp \ test-server/cpp-v2-server/aws-sdk-cpp From d8c767621a503d90ee0541f6807f34e14cd3d5a6 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 14:07:49 -0800 Subject: [PATCH 24/45] Update makefile --- test-server/Makefile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test-server/Makefile b/test-server/Makefile index 4a8e45ca..948a3ba5 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -8,7 +8,11 @@ export MAKEFLAGS=-j$(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || all: start-all-servers wait-all-servers run-tests # CI target for GitHub Actions -ci: start-all-servers wait-all-servers run-tests stop-servers +ci: + $(MAKE) start-all-servers + $(MAKE) wait-all-servers + $(MAKE) run-tests + $(MAKE) stop-servers SERVER_DIRS := $(shell find . -maxdepth 1 -type d -name '*-server' | sed 's|^\./||' | $(if $(FILTER),grep -E "$$(echo '$(FILTER)' | sed 's/,/|/g')",cat) | sort) @@ -99,8 +103,8 @@ wait-for-port: echo "❌ Error: PORT is required"; \ exit 1; \ fi - echo "Starting to wait for $$PORT to start"; \ - @for i in $$(seq 1 $(TIMEOUT)); do \ + @echo "Starting to wait for $$PORT to start"; \ + for i in $$(seq 1 $(TIMEOUT)); do \ if nc -z localhost $$PORT; then \ echo "Ports are open, waiting for servers to initialize..."; \ sleep 5; \ From b894708479824885ea65ee773f79b6fbfb6e3742 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 14:31:31 -0800 Subject: [PATCH 25/45] updates --- .github/workflows/test.yml | 12 +++++++----- test-server/cpp-v2-server/Makefile | 2 +- test-server/cpp-v2-transition-server/Makefile | 2 +- test-server/cpp-v3-server/Makefile | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4c36ed32..15d144df 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,16 +56,18 @@ jobs: git config --global fetch.parallel ${{ steps.cpu-count.outputs.count }} git config --global submodule.fetchJobs ${{ steps.cpu-count.outputs.count }} - - name: Checkout all submodules (use --jobs) + - name: Checkout submodules with --jobs + run: | + git submodule update --init --depth 1 --no-tags --jobs ${{ steps.cpu-count.outputs.count }} + + - name: Update cpp submodules recursively with --jobs run: | - # Non-recursive for most submodules - git submodule update --init --depth 1 --jobs ${{ steps.cpu-count.outputs.count }} - - # Recursive only for C++ SDK submodules with blobless clone git submodule update --init --recursive \ --depth 1 \ --filter=blob:none \ + --no-tags \ --jobs ${{ steps.cpu-count.outputs.count }} \ + --force \ test-server/cpp-v2-transition-server/aws-sdk-cpp \ test-server/cpp-v3-server/aws-sdk-cpp \ test-server/cpp-v2-server/aws-sdk-cpp diff --git a/test-server/cpp-v2-server/Makefile b/test-server/cpp-v2-server/Makefile index f060d569..646287a0 100644 --- a/test-server/cpp-v2-server/Makefile +++ b/test-server/cpp-v2-server/Makefile @@ -11,7 +11,7 @@ build/s3ec-server: start-server: | build/s3ec-server @echo "Building Cpp V2 server..." - cd build && make && \ + cd build && make @echo "Starting Cpp V2 server..." AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ diff --git a/test-server/cpp-v2-transition-server/Makefile b/test-server/cpp-v2-transition-server/Makefile index 82683cf4..d87b686f 100644 --- a/test-server/cpp-v2-transition-server/Makefile +++ b/test-server/cpp-v2-transition-server/Makefile @@ -10,7 +10,7 @@ build/s3ec-server: start-server: | build/s3ec-server @echo "Building Cpp transition server..." - cd build && make && \ + cd build && make @echo "Starting Cpp transition server..." AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ diff --git a/test-server/cpp-v3-server/Makefile b/test-server/cpp-v3-server/Makefile index 537eb402..397c7750 100644 --- a/test-server/cpp-v3-server/Makefile +++ b/test-server/cpp-v3-server/Makefile @@ -10,7 +10,7 @@ build/s3ec-server: start-server: | build/s3ec-server @echo "Building Cpp V3 server..." - cd build && make && \ + cd build && make @echo "Starting Cpp V3 server..." AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ From 4b11825288a16f324b6b23d5333e73cf7370bed7 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 14:37:27 -0800 Subject: [PATCH 26/45] move things around --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 15d144df..c33d9b4c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,17 +55,17 @@ jobs: run: | git config --global fetch.parallel ${{ steps.cpu-count.outputs.count }} git config --global submodule.fetchJobs ${{ steps.cpu-count.outputs.count }} + git config --global remote.origin.tagOpt --no-tags - name: Checkout submodules with --jobs run: | - git submodule update --init --depth 1 --no-tags --jobs ${{ steps.cpu-count.outputs.count }} + git submodule update --init --depth 1 --jobs ${{ steps.cpu-count.outputs.count }} - name: Update cpp submodules recursively with --jobs run: | git submodule update --init --recursive \ --depth 1 \ --filter=blob:none \ - --no-tags \ --jobs ${{ steps.cpu-count.outputs.count }} \ --force \ test-server/cpp-v2-transition-server/aws-sdk-cpp \ From 0f3dc82ecdf5dad05f71529ead94af01efa39cae Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 15:24:36 -0800 Subject: [PATCH 27/45] update the C makefiles --- test-server/cpp-v2-server/Makefile | 7 ++++--- test-server/cpp-v2-transition-server/Makefile | 7 ++++--- test-server/cpp-v3-server/Makefile | 7 ++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/test-server/cpp-v2-server/Makefile b/test-server/cpp-v2-server/Makefile index 646287a0..ab27da8b 100644 --- a/test-server/cpp-v2-server/Makefile +++ b/test-server/cpp-v2-server/Makefile @@ -11,13 +11,14 @@ build/s3ec-server: start-server: | build/s3ec-server @echo "Building Cpp V2 server..." - cd build && make - @echo "Starting Cpp V2 server..." + cd build && \ + make && \ + echo "Starting Cpp V2 server..." && \ AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./s3ec-server & echo $$! > $(PID_FILE) + ./s3ec-server & echo $$! > ../$(PID_FILE) @echo "Cpp V2 server starting..." stop-server: diff --git a/test-server/cpp-v2-transition-server/Makefile b/test-server/cpp-v2-transition-server/Makefile index d87b686f..5a13f16f 100644 --- a/test-server/cpp-v2-transition-server/Makefile +++ b/test-server/cpp-v2-transition-server/Makefile @@ -10,13 +10,14 @@ build/s3ec-server: start-server: | build/s3ec-server @echo "Building Cpp transition server..." - cd build && make - @echo "Starting Cpp transition server..." + cd build && \ + make && \ + echo "Starting Cpp transition server..." && \ AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./s3ec-server & echo $$! > $(PID_FILE) + ./s3ec-server & echo $$! > ../$(PID_FILE) @echo "Cpp transition server starting..." stop-server: diff --git a/test-server/cpp-v3-server/Makefile b/test-server/cpp-v3-server/Makefile index 397c7750..015d2ed1 100644 --- a/test-server/cpp-v3-server/Makefile +++ b/test-server/cpp-v3-server/Makefile @@ -10,13 +10,14 @@ build/s3ec-server: start-server: | build/s3ec-server @echo "Building Cpp V3 server..." - cd build && make - @echo "Starting Cpp V3 server..." + cd build && \ + make && \ + echo "Starting Cpp V3 server..." && \ AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./s3ec-server & echo $$! > $(PID_FILE) + ./s3ec-server & echo $$! > ../$(PID_FILE) @echo "Cpp V3 server starting..." stop-server: From bb9173a43ca7401e565614d38ecfa97eb5abf504 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 16:16:23 -0800 Subject: [PATCH 28/45] update the makefile continuation --- test-server/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-server/Makefile b/test-server/Makefile index 948a3ba5..8be30eb0 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -103,8 +103,8 @@ wait-for-port: echo "❌ Error: PORT is required"; \ exit 1; \ fi - @echo "Starting to wait for $$PORT to start"; \ - for i in $$(seq 1 $(TIMEOUT)); do \ + @echo "Starting to wait for $$PORT to start"; + @for i in $$(seq 1 $(TIMEOUT)); do \ if nc -z localhost $$PORT; then \ echo "Ports are open, waiting for servers to initialize..."; \ sleep 5; \ From d05dac70f91a9b2f69a4ea589918a3ea79b6556c Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 16:46:28 -0800 Subject: [PATCH 29/45] break out the commands --- test-server/cpp-v2-server/Makefile | 4 ++-- test-server/cpp-v2-transition-server/Makefile | 4 ++-- test-server/cpp-v3-server/Makefile | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test-server/cpp-v2-server/Makefile b/test-server/cpp-v2-server/Makefile index ab27da8b..a9bc218d 100644 --- a/test-server/cpp-v2-server/Makefile +++ b/test-server/cpp-v2-server/Makefile @@ -11,9 +11,9 @@ build/s3ec-server: start-server: | build/s3ec-server @echo "Building Cpp V2 server..." + cd build && make + @echo "Starting Cpp V2 server..." cd build && \ - make && \ - echo "Starting Cpp V2 server..." && \ AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ diff --git a/test-server/cpp-v2-transition-server/Makefile b/test-server/cpp-v2-transition-server/Makefile index 5a13f16f..e66096b5 100644 --- a/test-server/cpp-v2-transition-server/Makefile +++ b/test-server/cpp-v2-transition-server/Makefile @@ -10,9 +10,9 @@ build/s3ec-server: start-server: | build/s3ec-server @echo "Building Cpp transition server..." + cd build && make + @echo "Starting Cpp transition server..." cd build && \ - make && \ - echo "Starting Cpp transition server..." && \ AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ diff --git a/test-server/cpp-v3-server/Makefile b/test-server/cpp-v3-server/Makefile index 015d2ed1..9ac27c7a 100644 --- a/test-server/cpp-v3-server/Makefile +++ b/test-server/cpp-v3-server/Makefile @@ -10,9 +10,9 @@ build/s3ec-server: start-server: | build/s3ec-server @echo "Building Cpp V3 server..." + cd build && make + @echo "Starting Cpp V3 server..." cd build && \ - make && \ - echo "Starting Cpp V3 server..." && \ AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ From d75f1bc19e84a0045a2d5d0f30bf8a6d88c58655 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 17:07:13 -0800 Subject: [PATCH 30/45] try this redirect --- test-server/cpp-v2-server/Makefile | 2 +- test-server/cpp-v2-transition-server/Makefile | 2 +- test-server/cpp-v3-server/Makefile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test-server/cpp-v2-server/Makefile b/test-server/cpp-v2-server/Makefile index a9bc218d..75b8a1cf 100644 --- a/test-server/cpp-v2-server/Makefile +++ b/test-server/cpp-v2-server/Makefile @@ -18,7 +18,7 @@ start-server: | build/s3ec-server AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./s3ec-server & echo $$! > ../$(PID_FILE) + ./s3ec-server > server.log 2>&1 & echo $$! > ../$(PID_FILE) @echo "Cpp V2 server starting..." stop-server: diff --git a/test-server/cpp-v2-transition-server/Makefile b/test-server/cpp-v2-transition-server/Makefile index e66096b5..9d197c2e 100644 --- a/test-server/cpp-v2-transition-server/Makefile +++ b/test-server/cpp-v2-transition-server/Makefile @@ -17,7 +17,7 @@ start-server: | build/s3ec-server AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./s3ec-server & echo $$! > ../$(PID_FILE) + ./s3ec-server > server.log 2>&1 & echo $$! > ../$(PID_FILE) @echo "Cpp transition server starting..." stop-server: diff --git a/test-server/cpp-v3-server/Makefile b/test-server/cpp-v3-server/Makefile index 9ac27c7a..9371e729 100644 --- a/test-server/cpp-v3-server/Makefile +++ b/test-server/cpp-v3-server/Makefile @@ -17,7 +17,7 @@ start-server: | build/s3ec-server AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./s3ec-server & echo $$! > ../$(PID_FILE) + ./s3ec-server > server.log 2>&1 & echo $$! > ../$(PID_FILE) @echo "Cpp V3 server starting..." stop-server: From d2b30ee6e77d5447acc4ea50f71ece8a856aae5f Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 17:26:26 -0800 Subject: [PATCH 31/45] Break it apart --- .github/workflows/test.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c33d9b4c..563f34db 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -155,14 +155,31 @@ jobs: role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role aws-region: us-west-2 - - name: Run test-server tests - run: cd test-server && make ci FILTER=ruby,go,cpp + - name: Start the servers + run: cd test-server && make start-all-servers FILTER=ruby,go + env: + AWS_REGION: us-west-2 + TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} + TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} + + - name: Wait for servers to start + run: cd test-server && make wait-all-servers FILTER=ruby,go + env: + AWS_REGION: us-west-2 + TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} + TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} + + - name: Run run-tests + run: cd test-server && make run-tests FILTER=ruby,go env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} GRADLE_OPTS: "-Dorg.gradle.daemon=true -Dorg.gradle.parallel=true -Dorg.gradle.caching=true" + - name: Stop the servers + run: cd test-server && make stop-servers FILTER=ruby,go + - name: Upload results if: always() uses: actions/upload-artifact@v4 From 406c840a25d226f4877285cd21bf3646d144c962 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 11 Nov 2025 17:52:26 -0800 Subject: [PATCH 32/45] why? --- .github/workflows/test.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 563f34db..b9a2151f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -156,18 +156,18 @@ jobs: aws-region: us-west-2 - name: Start the servers - run: cd test-server && make start-all-servers FILTER=ruby,go + run: cd test-server && make start-servers FILTER=ruby,go env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - - name: Wait for servers to start - run: cd test-server && make wait-all-servers FILTER=ruby,go - env: - AWS_REGION: us-west-2 - TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} - TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} + # - name: Wait for servers to start + # run: cd test-server && make wait-all-servers FILTER=ruby,go + # env: + # AWS_REGION: us-west-2 + # TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} + # TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - name: Run run-tests run: cd test-server && make run-tests FILTER=ruby,go From 675cc90b6ba0b005492bde52e6422a4b6d1f5825 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Wed, 12 Nov 2025 09:57:56 -0800 Subject: [PATCH 33/45] lots of updates --- .github/workflows/test.yml | 29 ++++++++++++---- test-server/Makefile | 31 +++++++++++++---- test-server/cpp-v2-server/Makefile | 15 +++++--- test-server/cpp-v2-transition-server/Makefile | 15 +++++--- test-server/cpp-v3-server/Makefile | 15 +++++--- test-server/go-v3-server/Makefile | 18 +++++++--- test-server/go-v3-transition-server/Makefile | 18 +++++++--- test-server/go-v4-server/Makefile | 18 +++++++--- test-server/java-v3-server/Makefile | 17 +++++++--- .../java-v3-transition-server/Makefile | 21 ++++++++---- test-server/java-v4-server/Makefile | 21 ++++++++---- test-server/net-v2-v3-server/Makefile | 34 +++++++++++-------- test-server/net-v3-transition-server/Makefile | 21 ++++++++---- test-server/php-v2-server/Makefile | 17 +++++++--- test-server/php-v2-transition-server/Makefile | 17 +++++++--- test-server/php-v3-server/Makefile | 17 +++++++--- test-server/python-v3-server/Makefile | 20 +++++++---- test-server/ruby-v2-server/Makefile | 18 +++++++--- test-server/ruby-v3-server/Makefile | 18 +++++++--- 19 files changed, 273 insertions(+), 107 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b9a2151f..f5adcc91 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -155,19 +155,24 @@ jobs: role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role aws-region: us-west-2 + - name: Build the servers + run: cd test-server && make build-all-servers FILTER=ruby,go + env: + AWS_REGION: us-west-2 + - name: Start the servers - run: cd test-server && make start-servers FILTER=ruby,go + run: cd test-server && make start-all-servers FILTER=ruby,go env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - # - name: Wait for servers to start - # run: cd test-server && make wait-all-servers FILTER=ruby,go - # env: - # AWS_REGION: us-west-2 - # TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} - # TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} + - name: Wait for servers to start + run: cd test-server && make wait-all-servers FILTER=ruby,go + env: + AWS_REGION: us-west-2 + TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} + TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - name: Run run-tests run: cd test-server && make run-tests FILTER=ruby,go @@ -177,6 +182,16 @@ jobs: TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} GRADLE_OPTS: "-Dorg.gradle.daemon=true -Dorg.gradle.parallel=true -Dorg.gradle.caching=true" + - name: Upload server logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: server-logs + path: | + test-server/*/server.log + test-server/*/net-v2-server.log + test-server/*/net-v3-server.log + - name: Stop the servers run: cd test-server && make stop-servers FILTER=ruby,go diff --git a/test-server/Makefile b/test-server/Makefile index 8be30eb0..9b18b857 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -1,7 +1,5 @@ # Makefile for S3 Encryption Client Testing -export MAKEFLAGS=-j$(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 1) - .PHONY: all start-servers run-tests stop-servers clean ci check-env help # Default target @@ -9,6 +7,7 @@ all: start-all-servers wait-all-servers run-tests # CI target for GitHub Actions ci: + $(MAKE) build-all-servers $(MAKE) start-all-servers $(MAKE) wait-all-servers $(MAKE) run-tests @@ -16,11 +15,27 @@ ci: SERVER_DIRS := $(shell find . -maxdepth 1 -type d -name '*-server' | sed 's|^\./||' | $(if $(FILTER),grep -E "$$(echo '$(FILTER)' | sed 's/,/|/g')",cat) | sort) +BUILD_SERVER_TARGETS := $(addprefix build-, $(SERVER_DIRS)) START_SERVER_TARGETS := $(addprefix start-, $(SERVER_DIRS)) WAIT_SERVER_TARGETS := $(addprefix wait-, $(SERVER_DIRS)) -# Start all servers in parallel +# Build all servers in parallel +build-all-servers: export MAKEFLAGS=-j$(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 1) +build-all-servers: $(BUILD_SERVER_TARGETS) + +$(BUILD_SERVER_TARGETS): build-%: + @if [ -f $*/Makefile ]; then \ + echo "Building server in $*..."; \ + $(MAKE) -C $* build-server; \ + else \ + echo "❌ Error: no Makefile found in $*"; \ + exit 1; \ + fi + +# Build and start all servers start-servers: + @echo "Building all servers..." + $(MAKE) build-all-servers @echo "Starting all servers..." $(MAKE) start-all-servers @echo "Waiting for servers to start..." @@ -29,7 +44,9 @@ start-servers: $(MAKE) -C $$dir wait-for-server; \ done -start-all-servers: $(START_SERVER_TARGETS) +# Start servers sequentially (no parallel execution) +start-all-servers: + @$(MAKE) MAKEFLAGS= $(START_SERVER_TARGETS) $(START_SERVER_TARGETS): start-%: @if [ -f $*/Makefile ]; then \ @@ -38,7 +55,7 @@ $(START_SERVER_TARGETS): start-%: else \ echo "❌ Error: no Makefile found in $*"; \ exit 1; \ - fi; \ + fi; wait-all-servers: @echo "Waiting for all servers to be ready..." @@ -52,7 +69,7 @@ $(WAIT_SERVER_TARGETS): wait-%: else \ echo "❌ Error: no Makefile found in $*"; \ exit 1; \ - fi; \ + fi; # Run the Java tests @@ -72,7 +89,7 @@ run-tests: stop-servers: @echo "Stopping servers..." @for dir in $(SERVER_DIRS); do \ - echo "Starting server in $$dir..."; \ + echo "Stopping server in $$dir..."; \ $(MAKE) -C $$dir stop-server; \ done @echo "Servers stopped" diff --git a/test-server/cpp-v2-server/Makefile b/test-server/cpp-v2-server/Makefile index 75b8a1cf..77357c37 100644 --- a/test-server/cpp-v2-server/Makefile +++ b/test-server/cpp-v2-server/Makefile @@ -1,6 +1,6 @@ # Makefile for S3 Encryption Client Testing -.PHONY: start-server stop-server wait-for-server +.PHONY: build-server start-server stop-server wait-for-server PID_FILE := server.pid PORT := 8085 @@ -9,9 +9,11 @@ build/s3ec-server: cd aws-sdk-cpp mkdir -p build && cd build && cmake .. -start-server: | build/s3ec-server +build-server: | build/s3ec-server @echo "Building Cpp V2 server..." cd build && make + +start-server: @echo "Starting Cpp V2 server..." cd build && \ AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ @@ -22,10 +24,15 @@ start-server: | build/s3ec-server @echo "Cpp V2 server starting..." stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE) ]; then \ - kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ - rm $(PID_FILE); \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ fi + @rm -f build/server.log + @echo "Server stopped" wait-for-server: $(MAKE) -C .. wait-for-port PORT=$(PORT) diff --git a/test-server/cpp-v2-transition-server/Makefile b/test-server/cpp-v2-transition-server/Makefile index 9d197c2e..16b70796 100644 --- a/test-server/cpp-v2-transition-server/Makefile +++ b/test-server/cpp-v2-transition-server/Makefile @@ -1,6 +1,6 @@ # Makefile for S3 Encryption Client Testing -.PHONY: start-server stop-server wait-for-server +.PHONY: build-server start-server stop-server wait-for-server PID_FILE := server.pid PORT := 8097 @@ -8,9 +8,11 @@ PORT := 8097 build/s3ec-server: mkdir -p build && cd build && cmake .. -start-server: | build/s3ec-server +build-server: | build/s3ec-server @echo "Building Cpp transition server..." cd build && make + +start-server: @echo "Starting Cpp transition server..." cd build && \ AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ @@ -21,10 +23,15 @@ start-server: | build/s3ec-server @echo "Cpp transition server starting..." stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE) ]; then \ - kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ - rm $(PID_FILE); \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ fi + @rm -f build/server.log + @echo "Server stopped" wait-for-server: $(MAKE) -C .. wait-for-port PORT=$(PORT) diff --git a/test-server/cpp-v3-server/Makefile b/test-server/cpp-v3-server/Makefile index 9371e729..46f0c9db 100644 --- a/test-server/cpp-v3-server/Makefile +++ b/test-server/cpp-v3-server/Makefile @@ -1,6 +1,6 @@ # Makefile for S3 Encryption Client Testing -.PHONY: start-server stop-server wait-for-server +.PHONY: build-server start-server stop-server wait-for-server PID_FILE := server.pid PORT := 8091 @@ -8,9 +8,11 @@ PORT := 8091 build/s3ec-server: mkdir -p build && cd build && cmake .. -start-server: | build/s3ec-server +build-server: | build/s3ec-server @echo "Building Cpp V3 server..." cd build && make + +start-server: @echo "Starting Cpp V3 server..." cd build && \ AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ @@ -21,10 +23,15 @@ start-server: | build/s3ec-server @echo "Cpp V3 server starting..." stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE) ]; then \ - kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ - rm $(PID_FILE); \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ fi + @rm -f build/server.log + @echo "Server stopped" wait-for-server: $(MAKE) -C .. wait-for-port PORT=$(PORT) diff --git a/test-server/go-v3-server/Makefile b/test-server/go-v3-server/Makefile index fb61e578..80928dbd 100644 --- a/test-server/go-v3-server/Makefile +++ b/test-server/go-v3-server/Makefile @@ -1,25 +1,33 @@ # Makefile for S3 Encryption Client Testing -.PHONY: start-server stop-server wait-for-server +.PHONY: build-server start-server stop-server wait-for-server PID_FILE := server.pid PORT := 8082 +build-server: + @echo "Building Go V3 server..." + go mod tidy + start-server: @echo "Starting Go V3 server..." - go mod tidy AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - go run . & echo $$! > $(PID_FILE) + go run . > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "Go V3 server starting..." stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE) ]; then \ - kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ - rm $(PID_FILE); \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ fi + @rm -f server.log + @echo "Server stopped" wait-for-server: $(MAKE) -C .. wait-for-port PORT=$(PORT) diff --git a/test-server/go-v3-transition-server/Makefile b/test-server/go-v3-transition-server/Makefile index b03ea80b..a254acdf 100644 --- a/test-server/go-v3-transition-server/Makefile +++ b/test-server/go-v3-transition-server/Makefile @@ -1,25 +1,33 @@ # Makefile for S3 Encryption Client Testing -.PHONY: start-server stop-server wait-for-server +.PHONY: build-server start-server stop-server wait-for-server PID_FILE := server.pid PORT := 8095 +build-server: + @echo "Building Go V3 Transition server..." + go mod tidy + start-server: @echo "Starting Go V3 Transition server..." - go mod tidy AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - go run . & echo $$! > $(PID_FILE) + go run . > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "Go V3 Transition server starting..." stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE) ]; then \ - kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ - rm $(PID_FILE); \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ fi + @rm -f server.log + @echo "Server stopped" wait-for-server: $(MAKE) -C .. wait-for-port PORT=$(PORT) diff --git a/test-server/go-v4-server/Makefile b/test-server/go-v4-server/Makefile index cfaf32fe..6c549db2 100644 --- a/test-server/go-v4-server/Makefile +++ b/test-server/go-v4-server/Makefile @@ -1,25 +1,33 @@ # Makefile for S3 Encryption Client Testing -.PHONY: start-server stop-server wait-for-server +.PHONY: build-server start-server stop-server wait-for-server PID_FILE := server.pid PORT := 8089 +build-server: + @echo "Building Go V4 server..." + go mod tidy + start-server: @echo "Starting Go V4 server..." - go mod tidy AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - go run . & echo $$! > $(PID_FILE) + go run . > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "Go V4 server starting..." stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE) ]; then \ - kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ - rm $(PID_FILE); \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ fi + @rm -f server.log + @echo "Server stopped" wait-for-server: $(MAKE) -C .. wait-for-port PORT=$(PORT) diff --git a/test-server/java-v3-server/Makefile b/test-server/java-v3-server/Makefile index 445be2ac..692e80b3 100644 --- a/test-server/java-v3-server/Makefile +++ b/test-server/java-v3-server/Makefile @@ -1,24 +1,33 @@ # Makefile for S3 Encryption Client Testing -.PHONY: start-server stop-server wait-for-server +.PHONY: build-server start-server stop-server wait-for-server PID_FILE := server.pid PORT := 8080 +build-server: + @echo "Building Java V3 server..." + ./gradlew --build-cache --parallel build + start-server: @echo "Starting Java V3 server..." AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./gradlew --build-cache --parallel run & echo $$! > $(PID_FILE) + ./gradlew --build-cache --parallel run > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "Java V3 server starting..." stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE) ]; then \ - kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ - rm $(PID_FILE); \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ fi + @rm -f server.log + @echo "Server stopped" wait-for-server: $(MAKE) -C .. wait-for-port PORT=$(PORT) diff --git a/test-server/java-v3-transition-server/Makefile b/test-server/java-v3-transition-server/Makefile index 3f0358c9..5a25a8aa 100644 --- a/test-server/java-v3-transition-server/Makefile +++ b/test-server/java-v3-transition-server/Makefile @@ -1,29 +1,36 @@ # Makefile for S3 Encryption Client Testing -.PHONY: start-server stop-server wait-for-server build-s3ec +.PHONY: build-server start-server stop-server wait-for-server PID_FILE := server.pid PORT := 8094 -build-s3ec: +build-server: @echo "Building S3EC from source..." cd s3ec-staging && mvn --batch-mode -no-transfer-progress clean compile && mvn -B -ntp install -DskipTests @echo "S3EC build completed." + @echo "Building Java V3 Transition server..." + ./gradlew --build-cache --parallel build -start-server: build-s3ec +start-server: @echo "Starting Java V3 Transition server..." AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./gradlew --build-cache --parallel run & echo $$! > $(PID_FILE) + ./gradlew --build-cache --parallel run > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "Java V3 Transition server starting..." stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE) ]; then \ - kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ - rm $(PID_FILE); \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ fi + @rm -f server.log + @echo "Server stopped" wait-for-server: $(MAKE) -C .. wait-for-port PORT=$(PORT) @@ -32,4 +39,4 @@ duvet: duvet report view-report-mac: - open .duvet/reports/report.html \ No newline at end of file + open .duvet/reports/report.html diff --git a/test-server/java-v4-server/Makefile b/test-server/java-v4-server/Makefile index 734a7808..418e0127 100644 --- a/test-server/java-v4-server/Makefile +++ b/test-server/java-v4-server/Makefile @@ -1,29 +1,36 @@ # Makefile for S3 Encryption Client Testing -.PHONY: start-server stop-server wait-for-server build-s3ec +.PHONY: build-server start-server stop-server wait-for-server PID_FILE := server.pid PORT := 8088 -build-s3ec: +build-server: @echo "Building S3EC from source..." cd s3ec-staging && mvn --batch-mode -no-transfer-progress clean compile && mvn -B -ntp install -DskipTests @echo "S3EC build completed." + @echo "Building Java V4 server..." + ./gradlew --build-cache --parallel build -start-server: build-s3ec +start-server: @echo "Starting Java V4 server..." AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./gradlew --build-cache --parallel run & echo $$! > $(PID_FILE) + ./gradlew --build-cache --parallel run > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "Java V4 server starting..." stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE) ]; then \ - kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ - rm $(PID_FILE); \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ fi + @rm -f server.log + @echo "Server stopped" wait-for-server: $(MAKE) -C .. wait-for-port PORT=$(PORT) @@ -32,4 +39,4 @@ duvet: duvet report view-report-mac: - open .duvet/reports/report.html \ No newline at end of file + open .duvet/reports/report.html diff --git a/test-server/net-v2-v3-server/Makefile b/test-server/net-v2-v3-server/Makefile index e752b925..b50ae4f8 100644 --- a/test-server/net-v2-v3-server/Makefile +++ b/test-server/net-v2-v3-server/Makefile @@ -1,52 +1,58 @@ # Makefile for S3 Encryption Client .NET Testing -.PHONY: start-server stop-server wait-for-server +.PHONY: build-server start-server stop-server wait-for-server PID_FILE_NET_V2 := net-v2-server.pid PID_FILE_NET_V3 := net-v3-server.pid PORT_NET_V2 := 8083 PORT_NET_V3 := 8084 +build-server: + @echo "Building .NET V2 and V3 servers..." + rm -rf obj/v2 bin/v2 obj/v3 bin/v3 + dotnet build -p:S3EncryptionVersion=v2 -o bin/v2 -p:BaseIntermediateOutputPath=obj/v2/ + dotnet build -p:S3EncryptionVersion=v3 -o bin/v3 -p:BaseIntermediateOutputPath=obj/v3/ + start-server: $(MAKE) start-net-v2-server; \ $(MAKE) start-net-v3-server; stop-server: + @echo "Stopping .NET V2 server on port $(PORT_NET_V2)..." + @lsof -ti:$(PORT_NET_V2) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE_NET_V2) ]; then \ - kill $$(cat $(PID_FILE_NET_V2)) 2>/dev/null || true; \ - rm $(PID_FILE_NET_V2); \ + pkill -P $$(cat $(PID_FILE_NET_V2)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE_NET_V2)) 2>/dev/null || true; \ + rm -f $(PID_FILE_NET_V2); \ fi + @rm -f net-v2-server.log + @echo "Stopping .NET V3 server on port $(PORT_NET_V3)..." + @lsof -ti:$(PORT_NET_V3) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE_NET_V3) ]; then \ - kill $$(cat $(PID_FILE_NET_V3)) 2>/dev/null || true; \ - rm $(PID_FILE_NET_V3); \ + pkill -P $$(cat $(PID_FILE_NET_V3)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE_NET_V3)) 2>/dev/null || true; \ + rm -f $(PID_FILE_NET_V3); \ fi + @rm -f net-v3-server.log + @echo "Servers stopped" # Start .NET V2 server in background -# This builds first into bin/v2 and runs through dll -# to avoid simultaneous dotnet run conflict start-net-v2-server: @echo "Starting .NET V2 server..." AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - rm -rf obj/v2 bin/v2 && \ - dotnet build -p:S3EncryptionVersion=v2 -o bin/v2 -p:BaseIntermediateOutputPath=obj/v2/ && \ dotnet bin/v2/NetV2V3Server.dll > net-v2-server.log 2>&1 & echo $$! > net-v2-server.pid @echo ".NET V2 server starting..." - # Start .NET V3 server in background -# This builds first into bin/v3 and runs through dll -# to avoid simultaneous dotnet run conflict start-net-v3-server: @echo "Starting .NET V3 server..." AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - rm -rf obj/v3 bin/v3 && \ - dotnet build -p:S3EncryptionVersion=v3 -o bin/v3 -p:BaseIntermediateOutputPath=obj/v3/ && \ dotnet bin/v3/NetV2V3Server.dll > net-v3-server.log 2>&1 & echo $$! > net-v3-server.pid @echo ".NET V3 server starting..." diff --git a/test-server/net-v3-transition-server/Makefile b/test-server/net-v3-transition-server/Makefile index ca863f28..9e189eba 100644 --- a/test-server/net-v3-transition-server/Makefile +++ b/test-server/net-v3-transition-server/Makefile @@ -1,28 +1,37 @@ # Makefile for S3 Encryption Client .NET Testing -.PHONY: start-server stop-server wait-for-server +.PHONY: build-server start-server stop-server wait-for-server PID_FILE_NET_V3_TRANSITION := net-v3-transition-server.pid PORT_NET_V3_TRANSITION := 8100 +build-server: + @echo "Building .NET V3 transition server..." + dotnet build + start-server: $(MAKE) start-net-v3-transition-server stop-server: + @echo "Stopping .NET V3 Transition server on port $(PORT_NET_V3_TRANSITION)..." + @lsof -ti:$(PORT_NET_V3_TRANSITION) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE_NET_V3_TRANSITION) ]; then \ - kill $$(cat $(PID_FILE_NET_V3_TRANSITION)) 2>/dev/null || true; \ - rm $(PID_FILE_NET_V3_TRANSITION); \ + pkill -P $$(cat $(PID_FILE_NET_V3_TRANSITION)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE_NET_V3_TRANSITION)) 2>/dev/null || true; \ + rm -f $(PID_FILE_NET_V3_TRANSITION); \ fi + @rm -f server.log + @echo "Server stopped" # Start .NET V3 transition server in background start-net-v3-transition-server: - @echo "Starting .NET V3 server..." + @echo "Starting .NET V3 transition server..." AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - dotnet run & echo $! > net-v3-transition-server.pid - @echo ".NET V3 server starting..." + dotnet run > server.log 2>&1 & echo $$! > net-v3-transition-server.pid + @echo ".NET V3 transition server starting..." wait-for-server: $(MAKE) -C .. wait-for-port PORT=$(PORT_NET_V3_TRANSITION) diff --git a/test-server/php-v2-server/Makefile b/test-server/php-v2-server/Makefile index adb63258..a9d04134 100644 --- a/test-server/php-v2-server/Makefile +++ b/test-server/php-v2-server/Makefile @@ -1,24 +1,33 @@ # Makefile for S3 Encryption Client Testing -.PHONY: start-server stop-server wait-for-server +.PHONY: build-server start-server stop-server wait-for-server PID_FILE := server.pid PORT := 8087 +build-server: + @echo "Building PHP V2 server..." + composer install + start-server: @echo "Starting PHP V2 server..." AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - composer run start & echo $$! > $(PID_FILE) + composer run start > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "PHP V2 server starting..." stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE) ]; then \ - kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ - rm $(PID_FILE); \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ fi + @rm -f server.log + @echo "Server stopped" wait-for-server: $(MAKE) -C .. wait-for-port PORT=$(PORT) diff --git a/test-server/php-v2-transition-server/Makefile b/test-server/php-v2-transition-server/Makefile index 2544679d..61eb3a84 100644 --- a/test-server/php-v2-transition-server/Makefile +++ b/test-server/php-v2-transition-server/Makefile @@ -1,24 +1,33 @@ # Makefile for S3 Encryption Client Testing -.PHONY: start-server stop-server wait-for-server +.PHONY: build-server start-server stop-server wait-for-server PID_FILE := server.pid PORT := 8099 +build-server: + @echo "Building PHP V2 Transition server..." + composer install + start-server: @echo "Starting PHP V2 Transition server..." AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - composer run start & echo $$! > $(PID_FILE) + composer run start > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "PHP V2 Transition server starting..." stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE) ]; then \ - kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ - rm $(PID_FILE); \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ fi + @rm -f server.log + @echo "Server stopped" wait-for-server: $(MAKE) -C .. wait-for-port PORT=$(PORT) diff --git a/test-server/php-v3-server/Makefile b/test-server/php-v3-server/Makefile index 0ec40802..2b9661f2 100644 --- a/test-server/php-v3-server/Makefile +++ b/test-server/php-v3-server/Makefile @@ -1,24 +1,33 @@ # Makefile for S3 Encryption Client Testing -.PHONY: start-server stop-server wait-for-server +.PHONY: build-server start-server stop-server wait-for-server PID_FILE := server.pid PORT := 8093 +build-server: + @echo "Building PHP V3 server..." + composer install + start-server: @echo "Starting PHP V3 server..." AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - composer run start & echo $$! > $(PID_FILE) + composer run start > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "PHP V3 server starting..." stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE) ]; then \ - kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ - rm $(PID_FILE); \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ fi + @rm -f server.log + @echo "Server stopped" wait-for-server: $(MAKE) -C .. wait-for-port PORT=$(PORT) diff --git a/test-server/python-v3-server/Makefile b/test-server/python-v3-server/Makefile index 0468dc87..930c950c 100644 --- a/test-server/python-v3-server/Makefile +++ b/test-server/python-v3-server/Makefile @@ -1,28 +1,36 @@ # Makefile for S3 Encryption Client Testing -.PHONY: start-server stop-server wait-for-server +.PHONY: build-server start-server stop-server wait-for-server PID_FILE := server.pid PORT := 8081 -start-server: - @echo "Starting Python V3 server..." +build-server: + @echo "Building Python V3 server..." python -m venv .venv .venv/bin/python -m ensurepip .venv/bin/python -m pip install -e . .venv/bin/python -m pip install -e ../.. + +start-server: + @echo "Starting Python V3 server..." AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - .venv/bin/python src/main.py & echo $$! > $(PID_FILE) + .venv/bin/python src/main.py > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "Python V3 server starting..." stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE) ]; then \ - kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ - rm $(PID_FILE); \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ fi + @rm -f server.log + @echo "Server stopped" wait-for-server: $(MAKE) -C .. wait-for-port PORT=$(PORT) diff --git a/test-server/ruby-v2-server/Makefile b/test-server/ruby-v2-server/Makefile index f4297eac..e0f938fc 100644 --- a/test-server/ruby-v2-server/Makefile +++ b/test-server/ruby-v2-server/Makefile @@ -1,29 +1,37 @@ # Makefile for S3 Encryption Client Testing -.PHONY: start-server stop-server wait-for-server +.PHONY: build-server start-server stop-server wait-for-server PID_FILE := server.pid PORT := 8098 +build-server: + @echo "Building Ruby V2 server..." + bundle install + start-server: @if [ -f $(PID_FILE) ]; then \ echo "❌ Error: Server already running. Stop before starting."; \ exit 1; \ fi; @echo "Starting Ruby V2 server..." - bundle install AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - PORT=$(PORT) bundle exec ruby app.rb & echo $$! > $(PID_FILE) + PORT=$(PORT) bundle exec ruby app.rb > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "Ruby V2 server starting..." stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE) ]; then \ - kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ - rm $(PID_FILE); \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ fi + @rm -f server.log + @echo "Server stopped" wait-for-server: $(MAKE) -C .. wait-for-port PORT=$(PORT) diff --git a/test-server/ruby-v3-server/Makefile b/test-server/ruby-v3-server/Makefile index ec463bad..331abac5 100644 --- a/test-server/ruby-v3-server/Makefile +++ b/test-server/ruby-v3-server/Makefile @@ -1,29 +1,37 @@ # Makefile for S3 Encryption Client Testing -.PHONY: start-server stop-server wait-for-server +.PHONY: build-server start-server stop-server wait-for-server PID_FILE := server.pid PORT := 8092 +build-server: + @echo "Building Ruby V3 server..." + bundle install + start-server: @if [ -f $(PID_FILE) ]; then \ echo "❌ Error: Server already running. Stop before starting."; \ exit 1; \ fi; @echo "Starting Ruby V3 server..." - bundle install AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - PORT=$(PORT) bundle exec ruby app.rb & echo $$! > $(PID_FILE) + PORT=$(PORT) bundle exec ruby app.rb > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "Ruby V3 server starting..." stop-server: + @echo "Stopping server on port $(PORT)..." + @lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE) ]; then \ - kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ - rm $(PID_FILE); \ + pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm -f $(PID_FILE); \ fi + @rm -f server.log + @echo "Server stopped" wait-for-server: $(MAKE) -C .. wait-for-port PORT=$(PORT) From 9af5721395a5ef3b25b99c0804c6cd193288b515 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Wed, 12 Nov 2025 10:06:54 -0800 Subject: [PATCH 34/45] fast build? --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f5adcc91..461cc292 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -158,6 +158,7 @@ jobs: - name: Build the servers run: cd test-server && make build-all-servers FILTER=ruby,go env: + MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }} AWS_REGION: us-west-2 - name: Start the servers From be625453b8e8388769c8c6892f45c6d390061667 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Wed, 12 Nov 2025 10:31:22 -0800 Subject: [PATCH 35/45] add cpp back something --- .github/workflows/test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 461cc292..e2c43191 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -156,27 +156,27 @@ jobs: aws-region: us-west-2 - name: Build the servers - run: cd test-server && make build-all-servers FILTER=ruby,go + run: cd test-server && make build-all-servers FILTER=ruby,go,cpp env: MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }} AWS_REGION: us-west-2 - name: Start the servers - run: cd test-server && make start-all-servers FILTER=ruby,go + run: cd test-server && make start-all-servers FILTER=ruby,go,cpp env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - name: Wait for servers to start - run: cd test-server && make wait-all-servers FILTER=ruby,go + run: cd test-server && make wait-all-servers FILTER=ruby,go,cpp env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - name: Run run-tests - run: cd test-server && make run-tests FILTER=ruby,go + run: cd test-server && make run-tests FILTER=ruby,go,cpp env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} @@ -194,7 +194,7 @@ jobs: test-server/*/net-v3-server.log - name: Stop the servers - run: cd test-server && make stop-servers FILTER=ruby,go + run: cd test-server && make stop-servers FILTER=ruby,go,cpp - name: Upload results if: always() From 67045d408aebcf1e109e28c3eb39a5ab43c06cf4 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Wed, 12 Nov 2025 13:05:46 -0800 Subject: [PATCH 36/45] Update PHP --- test-server/php-v2-transition-server/local-php-sdk | 2 +- test-server/php-v3-server/local-php-sdk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test-server/php-v2-transition-server/local-php-sdk b/test-server/php-v2-transition-server/local-php-sdk index 35a52086..8e22105d 160000 --- a/test-server/php-v2-transition-server/local-php-sdk +++ b/test-server/php-v2-transition-server/local-php-sdk @@ -1 +1 @@ -Subproject commit 35a52086c5ccf7f5e62e3c17e210923e129c823b +Subproject commit 8e22105d4e10d3410a0d3390b18bad74fe21be00 diff --git a/test-server/php-v3-server/local-php-sdk b/test-server/php-v3-server/local-php-sdk index 35a52086..8e22105d 160000 --- a/test-server/php-v3-server/local-php-sdk +++ b/test-server/php-v3-server/local-php-sdk @@ -1 +1 @@ -Subproject commit 35a52086c5ccf7f5e62e3c17e210923e129c823b +Subproject commit 8e22105d4e10d3410a0d3390b18bad74fe21be00 From 7875a529cee3332f963f2e350e3fd0ae91e6a684 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Wed, 12 Nov 2025 13:07:47 -0800 Subject: [PATCH 37/45] Add PHP --- .github/workflows/test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e2c43191..febccd0c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -156,27 +156,27 @@ jobs: aws-region: us-west-2 - name: Build the servers - run: cd test-server && make build-all-servers FILTER=ruby,go,cpp + run: cd test-server && make build-all-servers FILTER=ruby,go,cpp,php env: MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }} AWS_REGION: us-west-2 - name: Start the servers - run: cd test-server && make start-all-servers FILTER=ruby,go,cpp + run: cd test-server && make start-all-servers FILTER=ruby,go,cpp,php env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - name: Wait for servers to start - run: cd test-server && make wait-all-servers FILTER=ruby,go,cpp + run: cd test-server && make wait-all-servers FILTER=ruby,go,cpp,php env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - name: Run run-tests - run: cd test-server && make run-tests FILTER=ruby,go,cpp + run: cd test-server && make run-tests FILTER=ruby,go,cpp,php env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} @@ -194,7 +194,7 @@ jobs: test-server/*/net-v3-server.log - name: Stop the servers - run: cd test-server && make stop-servers FILTER=ruby,go,cpp + run: cd test-server && make stop-servers FILTER=ruby,go,cpp,php - name: Upload results if: always() From 30a7aeaa66055fd1062479548603ddccc1026211 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Wed, 12 Nov 2025 16:49:42 -0800 Subject: [PATCH 38/45] add .net --- .github/workflows/test.yml | 10 +++++----- .../net-v3-transition-server/s3ec-v3-transition-branch | 2 +- test-server/net-v4-server/s3ec-net-v4-improved | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index febccd0c..109ae976 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -156,27 +156,27 @@ jobs: aws-region: us-west-2 - name: Build the servers - run: cd test-server && make build-all-servers FILTER=ruby,go,cpp,php + run: cd test-server && make build-all-servers FILTER=ruby,go,cpp,php,net env: MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }} AWS_REGION: us-west-2 - name: Start the servers - run: cd test-server && make start-all-servers FILTER=ruby,go,cpp,php + run: cd test-server && make start-all-servers FILTER=ruby,go,cpp,php,net env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - name: Wait for servers to start - run: cd test-server && make wait-all-servers FILTER=ruby,go,cpp,php + run: cd test-server && make wait-all-servers FILTER=ruby,go,cpp,php,net env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - name: Run run-tests - run: cd test-server && make run-tests FILTER=ruby,go,cpp,php + run: cd test-server && make run-tests FILTER=ruby,go,cpp,php,net env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} @@ -194,7 +194,7 @@ jobs: test-server/*/net-v3-server.log - name: Stop the servers - run: cd test-server && make stop-servers FILTER=ruby,go,cpp,php + run: cd test-server && make stop-servers FILTER=ruby,go,cpp,php,net - name: Upload results if: always() diff --git a/test-server/net-v3-transition-server/s3ec-v3-transition-branch b/test-server/net-v3-transition-server/s3ec-v3-transition-branch index 56008baf..ae9327f0 160000 --- a/test-server/net-v3-transition-server/s3ec-v3-transition-branch +++ b/test-server/net-v3-transition-server/s3ec-v3-transition-branch @@ -1 +1 @@ -Subproject commit 56008baf1ef63b084a01a30db69af32e870a655b +Subproject commit ae9327f0e21999ac263bc82ae553839230ee9117 diff --git a/test-server/net-v4-server/s3ec-net-v4-improved b/test-server/net-v4-server/s3ec-net-v4-improved index 691d22a5..5178af52 160000 --- a/test-server/net-v4-server/s3ec-net-v4-improved +++ b/test-server/net-v4-server/s3ec-net-v4-improved @@ -1 +1 @@ -Subproject commit 691d22a504184fd71f2dae7fd354bd669b58cc07 +Subproject commit 5178af527160bdb66cdbee4d04faa900bf8032f7 From fbfc2fe8a16890718f85865f36b2eb1d374c9e05 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Wed, 12 Nov 2025 17:10:43 -0800 Subject: [PATCH 39/45] only net? --- .github/workflows/test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 109ae976..1b50bab2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -156,27 +156,27 @@ jobs: aws-region: us-west-2 - name: Build the servers - run: cd test-server && make build-all-servers FILTER=ruby,go,cpp,php,net + run: cd test-server && make build-all-servers FILTER=net env: MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }} AWS_REGION: us-west-2 - name: Start the servers - run: cd test-server && make start-all-servers FILTER=ruby,go,cpp,php,net + run: cd test-server && make start-all-servers FILTER=net env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - name: Wait for servers to start - run: cd test-server && make wait-all-servers FILTER=ruby,go,cpp,php,net + run: cd test-server && make wait-all-servers FILTER=net env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - name: Run run-tests - run: cd test-server && make run-tests FILTER=ruby,go,cpp,php,net + run: cd test-server && make run-tests FILTER=net env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} @@ -194,7 +194,7 @@ jobs: test-server/*/net-v3-server.log - name: Stop the servers - run: cd test-server && make stop-servers FILTER=ruby,go,cpp,php,net + run: cd test-server && make stop-servers FILTER=net - name: Upload results if: always() From 0346f621c349a7d0b45aca2284b60a89eb47017d Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Wed, 12 Nov 2025 17:37:48 -0800 Subject: [PATCH 40/45] try net --- test-server/net-v3-transition-server/Makefile | 2 +- test-server/net-v4-server/Makefile | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/test-server/net-v3-transition-server/Makefile b/test-server/net-v3-transition-server/Makefile index 9e189eba..eba78e1c 100644 --- a/test-server/net-v3-transition-server/Makefile +++ b/test-server/net-v3-transition-server/Makefile @@ -30,7 +30,7 @@ start-net-v3-transition-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - dotnet run > server.log 2>&1 & echo $$! > net-v3-transition-server.pid + dotnet run > server.log 2>&1 & echo $$! > $(PID_FILE_NET_V3_TRANSITION) @echo ".NET V3 transition server starting..." wait-for-server: diff --git a/test-server/net-v4-server/Makefile b/test-server/net-v4-server/Makefile index 49e4db32..ad9c627a 100644 --- a/test-server/net-v4-server/Makefile +++ b/test-server/net-v4-server/Makefile @@ -5,14 +5,23 @@ PID_FILE_NET_V4 := net-V4-server.pid PORT_NET_V4 := 8090 +build-server: + @echo "Building .NET V4 improved server..." + dotnet build + start-server: $(MAKE) start-net-V4-server; stop-server: + @echo "Stopping .NET V4 Improved server on port $(PORT_NET_V4)..." + @lsof -ti:$(PORT_NET_V4) | xargs kill -9 2>/dev/null || true @if [ -f $(PID_FILE_NET_V4) ]; then \ - kill $$(cat $(PID_FILE_NET_V4)) 2>/dev/null || true; \ - rm $(PID_FILE_NET_V4); \ + pkill -P $$(cat $(PID_FILE_NET_V4)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE_NET_V4)) 2>/dev/null || true; \ + rm -f $(PID_FILE_NET_V4); \ fi + @rm -f server.log + @echo "Server stopped" # Start .NET V4 server in background # This builds first into bin/V4 and runs through dll @@ -23,7 +32,7 @@ start-net-V4-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - dotnet run & echo $! > net-v3-transition-server.pid + dotnet run & echo $! > $(PID_FILE_NET_V4) @echo ".NET V4 server starting..." wait-for-server: From 82ee608878f432dbe3c69e432a64f4529b1dc1bb Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Thu, 13 Nov 2025 09:09:00 -0800 Subject: [PATCH 41/45] try this? --- test-server/net-v2-v3-server/NetV2V3Server.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-server/net-v2-v3-server/NetV2V3Server.csproj b/test-server/net-v2-v3-server/NetV2V3Server.csproj index 8d664eff..a3da397e 100644 --- a/test-server/net-v2-v3-server/NetV2V3Server.csproj +++ b/test-server/net-v2-v3-server/NetV2V3Server.csproj @@ -28,7 +28,7 @@ - + From c185b6e22c410f63125278cefe6b3828b8a2e9b0 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Thu, 13 Nov 2025 09:52:41 -0800 Subject: [PATCH 42/45] just move forward --- .github/workflows/test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1b50bab2..febccd0c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -156,27 +156,27 @@ jobs: aws-region: us-west-2 - name: Build the servers - run: cd test-server && make build-all-servers FILTER=net + run: cd test-server && make build-all-servers FILTER=ruby,go,cpp,php env: MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }} AWS_REGION: us-west-2 - name: Start the servers - run: cd test-server && make start-all-servers FILTER=net + run: cd test-server && make start-all-servers FILTER=ruby,go,cpp,php env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - name: Wait for servers to start - run: cd test-server && make wait-all-servers FILTER=net + run: cd test-server && make wait-all-servers FILTER=ruby,go,cpp,php env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - name: Run run-tests - run: cd test-server && make run-tests FILTER=net + run: cd test-server && make run-tests FILTER=ruby,go,cpp,php env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} @@ -194,7 +194,7 @@ jobs: test-server/*/net-v3-server.log - name: Stop the servers - run: cd test-server && make stop-servers FILTER=net + run: cd test-server && make stop-servers FILTER=ruby,go,cpp,php - name: Upload results if: always() From 52f037f71a6408cf558fe2568c636ef81aa9f057 Mon Sep 17 00:00:00 2001 From: seebees Date: Thu, 13 Nov 2025 10:05:42 -0800 Subject: [PATCH 43/45] Update test-server/net-v4-server/Makefile Co-authored-by: Rishav karanjit --- test-server/net-v4-server/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-server/net-v4-server/Makefile b/test-server/net-v4-server/Makefile index ad9c627a..e2df658a 100644 --- a/test-server/net-v4-server/Makefile +++ b/test-server/net-v4-server/Makefile @@ -32,7 +32,7 @@ start-net-V4-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - dotnet run & echo $! > $(PID_FILE_NET_V4) + dotnet run --no-build & echo $! > $(PID_FILE_NET_V4) @echo ".NET V4 server starting..." wait-for-server: From b977487d15fbc64baf185c303d0daf7e8c09ec97 Mon Sep 17 00:00:00 2001 From: seebees Date: Thu, 13 Nov 2025 10:05:48 -0800 Subject: [PATCH 44/45] Update test-server/net-v2-v3-server/NetV2V3Server.csproj Co-authored-by: Rishav karanjit --- test-server/net-v2-v3-server/NetV2V3Server.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-server/net-v2-v3-server/NetV2V3Server.csproj b/test-server/net-v2-v3-server/NetV2V3Server.csproj index a3da397e..8d664eff 100644 --- a/test-server/net-v2-v3-server/NetV2V3Server.csproj +++ b/test-server/net-v2-v3-server/NetV2V3Server.csproj @@ -28,7 +28,7 @@ - + From 931bd4fff07da459845cb249affbdda00582eb54 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Thu, 13 Nov 2025 10:08:39 -0800 Subject: [PATCH 45/45] remove php --- .github/workflows/test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index febccd0c..e2c43191 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -156,27 +156,27 @@ jobs: aws-region: us-west-2 - name: Build the servers - run: cd test-server && make build-all-servers FILTER=ruby,go,cpp,php + run: cd test-server && make build-all-servers FILTER=ruby,go,cpp env: MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }} AWS_REGION: us-west-2 - name: Start the servers - run: cd test-server && make start-all-servers FILTER=ruby,go,cpp,php + run: cd test-server && make start-all-servers FILTER=ruby,go,cpp env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - name: Wait for servers to start - run: cd test-server && make wait-all-servers FILTER=ruby,go,cpp,php + run: cd test-server && make wait-all-servers FILTER=ruby,go,cpp env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - name: Run run-tests - run: cd test-server && make run-tests FILTER=ruby,go,cpp,php + run: cd test-server && make run-tests FILTER=ruby,go,cpp env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} @@ -194,7 +194,7 @@ jobs: test-server/*/net-v3-server.log - name: Stop the servers - run: cd test-server && make stop-servers FILTER=ruby,go,cpp,php + run: cd test-server && make stop-servers FILTER=ruby,go,cpp - name: Upload results if: always()