From c4e15629c93623af68ebef03609af9b7e3d54d24 Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Fri, 18 Jul 2025 16:28:08 +0200 Subject: [PATCH 1/6] Upload to PyPI --- .github/workflows/on_external_dispatch.yml | 18 ++-- .github/workflows/upload_to_pypi.yml | 103 +++++++++++++++++++++ 2 files changed, 112 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/upload_to_pypi.yml diff --git a/.github/workflows/on_external_dispatch.yml b/.github/workflows/on_external_dispatch.yml index 33a807db..d863063e 100644 --- a/.github/workflows/on_external_dispatch.yml +++ b/.github/workflows/on_external_dispatch.yml @@ -1,4 +1,4 @@ -name: Builds triggered externally by DuckDB +name: External Dispatch on: workflow_dispatch: inputs: @@ -12,7 +12,7 @@ on: required: false publish_packages: type: boolean - description: Publish packages on S3 and PyPI? + description: Publish to S3 required: true default: false @@ -27,8 +27,8 @@ jobs: duckdb_git_ref: ${{ inputs.duckdb-sha }} force_version: ${{ inputs.force_version }} - upload_to_staging: - name: Upload Artifacts to staging + publish-s3: + name: Publish Artifacts to the S3 Staging Bucket runs-on: ubuntu-latest needs: [ externally_triggered_build ] if: ${{ github.repository_owner == 'duckdb' && inputs.publish_packages }} @@ -48,10 +48,10 @@ jobs: aws-secret-access-key: ${{ secrets.S3_DUCKDB_STAGING_KEY }} - name: Upload artifacts to S3 bucket + # semantics: if a version is forced then we upload into a folder by the version name, otherwise we upload + # into a folder that is named -. Only the latter will be discovered be + # upload_to_pypi.yml. shell: bash run: | - DUCKDB_SHA="${{ inputs.duckdb-sha }}" - aws s3 cp \ - artifacts \ - s3://duckdb-staging/${DUCKDB_SHA:0:10}/${{ github.repository }}/ \ - --recursive + FOLDER="${{ inputs.force_version != '' && inputs.force_version || format('{0}-{1}', github.run_id, github.run_attempt) }}" + aws s3 cp artifacts s3://duckdb-staging/${{ github.repository }}/${FOLDER}/ --recursive diff --git a/.github/workflows/upload_to_pypi.yml b/.github/workflows/upload_to_pypi.yml new file mode 100644 index 00000000..48d0ec02 --- /dev/null +++ b/.github/workflows/upload_to_pypi.yml @@ -0,0 +1,103 @@ +name: upload_to_pypi.yml +on: + # this workflow runs after the below workflows are completed + workflow_run: + workflows: [ External Dispatch ] + types: [ completed ] + branches: + - main + - v*.*-* + workflow_dispatch: + inputs: + environment: + description: Environment to run in () + type: choice + required: true + default: test.pypi + options: + - test.pypi + - production.pypi + artifact_folder: + description: The S3 folder that contains the artifacts (s3://duckdb-staging/duckdb/duckdb-python/) + type: string + required: true + +jobs: + prepare: + name: Prepare and guard upload + if: ${{ github.repository_owner == 'duckdb' && ( github.event.workflow_run.conclusion == 'success' || github.event_name != 'workflow_run' ) }} + runs-on: ubuntu-latest + outputs: + s3_prefix: ${{ steps.get_s3_prefix.outputs.s3_prefix }} + steps: + - name: Determine S3 Prefix + id: get_s3_prefix + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + artifact_folder="${{ inputs.artifact_folder }}" + elif [[ -n "${{ github.event.workflow_run.id }}" && -n "${{ github.event.workflow_run.run_attempt }}" ]]; then + artifact_folder="${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}" + fi + if [[ -n "${artifact_folder}" ]]; then + s3_prefix="${{ github.repository }}/${artifact_folder}" + echo "Created S3 prefix: ${s3_prefix}" + echo "s3_prefix=${s3_prefix}" >> $GITHUB_OUTPUT + else + echo "Can't determine S3 prefix for event: ${{ github.event_name }}. Quitting." + exit 1 + fi + + - name: Authenticate With AWS + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: 'us-east-2' + aws-access-key-id: ${{ secrets.S3_DUCKDB_STAGING_ID }} + aws-secret-access-key: ${{ secrets.S3_DUCKDB_STAGING_KEY }} + + - name: Check S3 Prefix + shell: bash + run: | + if [[ $(aws s3api list-objects-v2 \ + --bucket duckdb-staging \ + --prefix "${{ steps.get_s3_prefix.outputs.s3_prefix }}/" \ + --max-items 1 \ + --query 'Contents[0].Key' \ + --output text) == "None" ]]; then + echo "Prefix does not exist: ${{ steps.get_s3_prefix.outputs.s3_prefix }}" + echo "${{ github.event_name == 'workflow_run' && 'Possibly built a stable release?' || 'Unexpected error' }}" + exit 1 + fi + + publish-pypi: + name: Publish Artifacts to PyPI + needs: [ prepare ] + runs-on: ubuntu-latest + environment: + name: ${{ github.event_name == 'workflow_dispatch' && inputs.environment || 'test.pypi' }} + if: ${{ vars.PYPI_URL != '' }} + permissions: + # this is needed for the OIDC flow that is used with trusted publishing on PyPI + id-token: write + steps: + - name: Authenticate With AWS + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: 'us-east-2' + aws-access-key-id: ${{ secrets.S3_DUCKDB_STAGING_ID }} + aws-secret-access-key: ${{ secrets.S3_DUCKDB_STAGING_KEY }} + + - name: Download Artifacts From S3 + env: + S3_URL: 's3://duckdb-staging/${{ needs.prepare.outputs.s3_prefix }}/' + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} + run: | + mkdir packages + aws s3 cp --recursive "${S3_URL}" packages + + - name: Upload artifacts to PyPI + if: ${{ vars.PYPI_URL != '' }} + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: ${{ vars.PYPI_URL }} + packages-dir: packages From 87ab43ff7da0715a6826780204782b1b1e821362 Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Fri, 18 Jul 2025 22:30:03 +0200 Subject: [PATCH 2/6] Debugging upload --- .github/workflows/upload_to_pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upload_to_pypi.yml b/.github/workflows/upload_to_pypi.yml index 48d0ec02..e239b158 100644 --- a/.github/workflows/upload_to_pypi.yml +++ b/.github/workflows/upload_to_pypi.yml @@ -74,7 +74,7 @@ jobs: runs-on: ubuntu-latest environment: name: ${{ github.event_name == 'workflow_dispatch' && inputs.environment || 'test.pypi' }} - if: ${{ vars.PYPI_URL != '' }} + #if: ${{ vars.PYPI_URL != '' }} permissions: # this is needed for the OIDC flow that is used with trusted publishing on PyPI id-token: write From fb8a993cd73b374779614dabc839e9adc512e685 Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Fri, 18 Jul 2025 22:46:19 +0200 Subject: [PATCH 3/6] Small pypi upload fixes --- .github/workflows/upload_to_pypi.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/upload_to_pypi.yml b/.github/workflows/upload_to_pypi.yml index e239b158..2c933436 100644 --- a/.github/workflows/upload_to_pypi.yml +++ b/.github/workflows/upload_to_pypi.yml @@ -22,6 +22,8 @@ on: type: string required: true +concurrency: ${{ inputs.artifact_folder || format('{0}-{1}', github.event.workflow_run.id, github.event.workflow_run.run_attempt) }} + jobs: prepare: name: Prepare and guard upload @@ -33,11 +35,7 @@ jobs: - name: Determine S3 Prefix id: get_s3_prefix run: | - if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - artifact_folder="${{ inputs.artifact_folder }}" - elif [[ -n "${{ github.event.workflow_run.id }}" && -n "${{ github.event.workflow_run.run_attempt }}" ]]; then - artifact_folder="${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}" - fi + artifact_folder="${{ inputs.artifact_folder || format('{0}-{1}', github.event.workflow_run.id, github.event.workflow_run.run_attempt) }}" if [[ -n "${artifact_folder}" ]]; then s3_prefix="${{ github.repository }}/${artifact_folder}" echo "Created S3 prefix: ${s3_prefix}" @@ -74,11 +72,18 @@ jobs: runs-on: ubuntu-latest environment: name: ${{ github.event_name == 'workflow_dispatch' && inputs.environment || 'test.pypi' }} - #if: ${{ vars.PYPI_URL != '' }} permissions: # this is needed for the OIDC flow that is used with trusted publishing on PyPI id-token: write steps: + - name: Fail if PYPI_URL is not set + if: ${{ vars.PYPI_URL == '' }} + shell: bash + run: | + env_name="${{ github.event_name == 'workflow_dispatch' && inputs.environment || 'test.pypi' }}" + echo "Error: vars.PYPI_URL is not set in the resolved environment (${env_name})" + exit 1 + - name: Authenticate With AWS uses: aws-actions/configure-aws-credentials@v4 with: @@ -96,7 +101,6 @@ jobs: aws s3 cp --recursive "${S3_URL}" packages - name: Upload artifacts to PyPI - if: ${{ vars.PYPI_URL != '' }} uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: ${{ vars.PYPI_URL }} From cff3a566a7b8cc2fe0723c15b8df98ae0db0bf43 Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Mon, 21 Jul 2025 11:03:12 +0200 Subject: [PATCH 4/6] Bump submodule --- external/duckdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/duckdb b/external/duckdb index 0b83e5d2..c1c3d888 160000 --- a/external/duckdb +++ b/external/duckdb @@ -1 +1 @@ -Subproject commit 0b83e5d2f68bc02dfefde74b846bd039f078affa +Subproject commit c1c3d88864ff367936b7c667e0ac071bceed5215 From 2b81659afa9196e5c06cbebf1526aa57f84e48b8 Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Mon, 21 Jul 2025 11:25:38 +0200 Subject: [PATCH 5/6] Push given duckdb sha as submodule ref --- .github/workflows/on_external_dispatch.yml | 41 +++++++++++++++++-- .github/workflows/on_pr.yml | 2 +- .github/workflows/on_push_postrelease.yml | 2 +- .../{pypi_packaging.yml => packaging.yml} | 0 4 files changed, 40 insertions(+), 5 deletions(-) rename .github/workflows/{pypi_packaging.yml => packaging.yml} (100%) diff --git a/.github/workflows/on_external_dispatch.yml b/.github/workflows/on_external_dispatch.yml index d863063e..de751cf4 100644 --- a/.github/workflows/on_external_dispatch.yml +++ b/.github/workflows/on_external_dispatch.yml @@ -6,6 +6,10 @@ on: type: string description: The DuckDB SHA to build against required: true + commit_duckdb_sha: + type: boolean + description: Set (commit) the DuckDB submodule to the given SHA + default: false force_version: type: string description: Force version (vX.Y.Z-((rc|post)N)) @@ -14,12 +18,44 @@ on: type: boolean description: Publish to S3 required: true - default: false + default: true + +defaults: + run: + shell: bash jobs: + commit_duckdb_submodule_sha: + name: Commit the submodule to the given DuckDB sha + if: ${{ inputs.commit_duckdb_sha }} + runs-on: ubuntu-24.04 + steps: + + - name: Checkout DuckDB Python + uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: 0 + submodules: true + + - name: Checkout DuckDB + run: | + cd external/duckdb + git fetch origin + git checkout ${{ inputs.duckdb_git_ref }} + + - name: Commit and push new submodule ref + # see https://github.com/actions/checkout?tab=readme-ov-file#push-a-commit-to-a-pr-using-the-built-in-token + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add external/duckdb + git commit -m "Update submodule ref" + git push + externally_triggered_build: name: Build and test releases - uses: ./.github/workflows/pypi_packaging.yml + uses: ./.github/workflows/packaging.yml with: minimal: false testsuite: all @@ -51,7 +87,6 @@ jobs: # semantics: if a version is forced then we upload into a folder by the version name, otherwise we upload # into a folder that is named -. Only the latter will be discovered be # upload_to_pypi.yml. - shell: bash run: | FOLDER="${{ inputs.force_version != '' && inputs.force_version || format('{0}-{1}', github.run_id, github.run_attempt) }}" aws s3 cp artifacts s3://duckdb-staging/${{ github.repository }}/${FOLDER}/ --recursive diff --git a/.github/workflows/on_pr.yml b/.github/workflows/on_pr.yml index 81fd3247..fd7d0df6 100644 --- a/.github/workflows/on_pr.yml +++ b/.github/workflows/on_pr.yml @@ -24,7 +24,7 @@ jobs: name: Build a minimal set of packages and run all tests on them # Skip packaging tests for draft PRs if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} - uses: ./.github/workflows/pypi_packaging.yml + uses: ./.github/workflows/packaging.yml with: minimal: true testsuite: all diff --git a/.github/workflows/on_push_postrelease.yml b/.github/workflows/on_push_postrelease.yml index d12ccdf8..51dedaa6 100644 --- a/.github/workflows/on_push_postrelease.yml +++ b/.github/workflows/on_push_postrelease.yml @@ -33,7 +33,7 @@ jobs: packaging_test: name: Build and test post release packages and upload to S3 needs: extract_duckdb_tag - uses: ./.github/workflows/pypi_packaging.yml + uses: ./.github/workflows/packaging.yml with: minimal: false testsuite: all diff --git a/.github/workflows/pypi_packaging.yml b/.github/workflows/packaging.yml similarity index 100% rename from .github/workflows/pypi_packaging.yml rename to .github/workflows/packaging.yml From a05d25a827ac4c78fe6866dc4ec3c6b4a5cb602a Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Mon, 21 Jul 2025 12:51:28 +0200 Subject: [PATCH 6/6] Workflow naming and defaults fixes --- .github/workflows/on_external_dispatch.yml | 24 ++++++++--------- .github/workflows/on_pr.yml | 2 +- .github/workflows/on_push_postrelease.yml | 6 ++--- .github/workflows/packaging.yml | 30 +++++++++++----------- .github/workflows/upload_to_pypi.yml | 8 +++--- 5 files changed, 35 insertions(+), 35 deletions(-) diff --git a/.github/workflows/on_external_dispatch.yml b/.github/workflows/on_external_dispatch.yml index de751cf4..5ab7f8a7 100644 --- a/.github/workflows/on_external_dispatch.yml +++ b/.github/workflows/on_external_dispatch.yml @@ -6,19 +6,19 @@ on: type: string description: The DuckDB SHA to build against required: true - commit_duckdb_sha: + commit-duckdb-sha: type: boolean description: Set (commit) the DuckDB submodule to the given SHA - default: false - force_version: + default: true + force-version: type: string description: Force version (vX.Y.Z-((rc|post)N)) required: false - publish_packages: + publish-packages: type: boolean description: Publish to S3 required: true - default: true + default: false defaults: run: @@ -27,7 +27,7 @@ defaults: jobs: commit_duckdb_submodule_sha: name: Commit the submodule to the given DuckDB sha - if: ${{ inputs.commit_duckdb_sha }} + if: ${{ inputs.commit-duckdb-sha }} runs-on: ubuntu-24.04 steps: @@ -59,15 +59,15 @@ jobs: with: minimal: false testsuite: all - git_ref: ${{ github.ref }} - duckdb_git_ref: ${{ inputs.duckdb-sha }} - force_version: ${{ inputs.force_version }} + git-ref: ${{ github.ref }} + duckdb-git-ref: ${{ inputs.duckdb-sha }} + force-version: ${{ inputs.force-version }} - publish-s3: + publish_s3: name: Publish Artifacts to the S3 Staging Bucket runs-on: ubuntu-latest needs: [ externally_triggered_build ] - if: ${{ github.repository_owner == 'duckdb' && inputs.publish_packages }} + if: ${{ github.repository_owner == 'duckdb' && inputs.publish-packages }} steps: - name: Fetch artifacts uses: actions/download-artifact@v4 @@ -88,5 +88,5 @@ jobs: # into a folder that is named -. Only the latter will be discovered be # upload_to_pypi.yml. run: | - FOLDER="${{ inputs.force_version != '' && inputs.force_version || format('{0}-{1}', github.run_id, github.run_attempt) }}" + FOLDER="${{ inputs.force-version != '' && inputs.force-version || format('{0}-{1}', github.run_id, github.run_attempt) }}" aws s3 cp artifacts s3://duckdb-staging/${{ github.repository }}/${FOLDER}/ --recursive diff --git a/.github/workflows/on_pr.yml b/.github/workflows/on_pr.yml index fd7d0df6..d8bfe825 100644 --- a/.github/workflows/on_pr.yml +++ b/.github/workflows/on_pr.yml @@ -28,7 +28,7 @@ jobs: with: minimal: true testsuite: all - duckdb_git_ref: ${{ github.base_ref }} + duckdb-git-ref: ${{ github.base_ref }} coverage_test: name: Run coverage tests diff --git a/.github/workflows/on_push_postrelease.yml b/.github/workflows/on_push_postrelease.yml index 51dedaa6..14575754 100644 --- a/.github/workflows/on_push_postrelease.yml +++ b/.github/workflows/on_push_postrelease.yml @@ -37,6 +37,6 @@ jobs: with: minimal: false testsuite: all - git_ref: ${{ github.ref }} - duckdb_git_ref: ${{ needs.extract_duckdb_tag.outputs.duckdb_version }} - force_version: ${{ github.ref_name }} + git-ref: ${{ github.ref }} + duckdb-git-ref: ${{ needs.extract_duckdb_tag.outputs.duckdb_version }} + force-version: ${{ github.ref_name }} diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 84016645..977bc914 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -1,5 +1,5 @@ name: Packaging -run-name: Build ${{ inputs.minimal && 'minimal set of' || 'all' }} packages (version=${{ inputs.force_version != '' && inputs.force_version || 'dev' }}, tests=${{ inputs.testsuite }}, ref=${{ inputs.git_ref }}, duckdb ref=${{ inputs.duckdb_git_ref }}) +run-name: Build ${{ inputs.minimal && 'minimal set of' || 'all' }} packages (version=${{ inputs.force-version != '' && inputs.force-version || 'dev' }}, tests=${{ inputs.testsuite }}, ref=${{ inputs.git-ref }}, duckdb ref=${{ inputs.duckdb-git-ref }}) on: workflow_dispatch: inputs: @@ -16,16 +16,16 @@ on: - none - fast - all - git_ref: + git-ref: type: string description: Git ref of the DuckDB python package required: false - duckdb_git_ref: + duckdb-git-ref: type: string description: Git ref of DuckDB required: true default: refs/heads/main - force_version: + force-version: type: string description: Force version (vX.Y.Z-((rc|post)N)) required: false @@ -40,15 +40,15 @@ on: description: Testsuite to run (none, fast, all) required: true default: all - git_ref: + git-ref: type: string description: Git ref of the DuckDB python package required: false - duckdb_git_ref: + duckdb-git-ref: type: string description: Git ref of DuckDB required: false - force_version: + force-version: description: Force version (vX.Y.Z-((rc|post)N)) required: false type: string @@ -70,7 +70,7 @@ jobs: - name: Checkout DuckDB Python uses: actions/checkout@v4 with: - ref: ${{ inputs.git_ref }} + ref: ${{ inputs.git-ref }} fetch-depth: 0 submodules: true @@ -79,11 +79,11 @@ jobs: run: | cd external/duckdb git fetch origin - git checkout ${{ inputs.duckdb_git_ref }} + git checkout ${{ inputs.duckdb-git-ref }} - name: Set OVERRIDE_GIT_DESCRIBE - if: ${{ inputs.force_version != '' }} - run: echo "OVERRIDE_GIT_DESCRIBE=${{ inputs.force_version }}" >> $GITHUB_ENV + if: ${{ inputs.force-version != '' }} + run: echo "OVERRIDE_GIT_DESCRIBE=${{ inputs.force-version }}" >> $GITHUB_ENV - name: Install Astral UV uses: astral-sh/setup-uv@v6 @@ -153,7 +153,7 @@ jobs: - name: Checkout DuckDB Python uses: actions/checkout@v4 with: - ref: ${{ inputs.git_ref }} + ref: ${{ inputs.git-ref }} fetch-depth: 0 submodules: true @@ -162,12 +162,12 @@ jobs: run: | cd external/duckdb git fetch origin - git checkout ${{ inputs.duckdb_git_ref }} + git checkout ${{ inputs.duckdb-git-ref }} # Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds - name: Set OVERRIDE_GIT_DESCRIBE - if: ${{ inputs.force_version != '' }} - run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.force_version }}" >> $GITHUB_ENV + if: ${{ inputs.force-version != '' }} + run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.force-version }}" >> $GITHUB_ENV # Install Astral UV, which will be used as build-frontend for cibuildwheel - uses: astral-sh/setup-uv@v6 diff --git a/.github/workflows/upload_to_pypi.yml b/.github/workflows/upload_to_pypi.yml index 2c933436..bce4c315 100644 --- a/.github/workflows/upload_to_pypi.yml +++ b/.github/workflows/upload_to_pypi.yml @@ -17,12 +17,12 @@ on: options: - test.pypi - production.pypi - artifact_folder: - description: The S3 folder that contains the artifacts (s3://duckdb-staging/duckdb/duckdb-python/) + artifact-folder: + description: The S3 folder that contains the artifacts (s3://duckdb-staging/duckdb/duckdb-python/) type: string required: true -concurrency: ${{ inputs.artifact_folder || format('{0}-{1}', github.event.workflow_run.id, github.event.workflow_run.run_attempt) }} +concurrency: ${{ inputs.artifact-folder || format('{0}-{1}', github.event.workflow_run.id, github.event.workflow_run.run_attempt) }} jobs: prepare: @@ -35,7 +35,7 @@ jobs: - name: Determine S3 Prefix id: get_s3_prefix run: | - artifact_folder="${{ inputs.artifact_folder || format('{0}-{1}', github.event.workflow_run.id, github.event.workflow_run.run_attempt) }}" + artifact_folder="${{ inputs.artifact-folder || format('{0}-{1}', github.event.workflow_run.id, github.event.workflow_run.run_attempt) }}" if [[ -n "${artifact_folder}" ]]; then s3_prefix="${{ github.repository }}/${artifact_folder}" echo "Created S3 prefix: ${s3_prefix}"