ci: Add nightly canary release for python sdk

.github/workflows/publish-py-sdk-canary-scheduler.yaml

@@ -0,0 +1,47 @@
+# This workflow schedules the Python SDK canary publish workflow on a daily basis.
+# The actual publish still happens in `test-publish-py-sdk.yaml` so TestPyPI trusted
+# publishing only needs that one workflow configured as the publisher.
+
+name: Schedule Python SDK Canary Publish
+
+concurrency:
+  group: publish-py-sdk-canary-scheduler
+  cancel-in-progress: false
+
+on:
+  schedule:
+    - cron: "23 6 * * *"
+  workflow_dispatch:
+
+jobs:
+  dispatch-canary-publish:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      actions: write
+    steps:
+      - name: Dispatch TestPyPI publish workflow
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
+        with:
+          script: |
+            await github.rest.actions.createWorkflowDispatch({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: "test-publish-py-sdk.yaml",
+              ref: "main",
+              inputs: {
+                ref: "main",
+                release_type: "canary",
+                dry_run: "false",
+              },
+            });
+
+      - name: Summarize dispatch
+        run: |
+          {
+            echo "## Python SDK Canary Dispatch Queued"
+            echo
+            echo "- Workflow: \`test-publish-py-sdk.yaml\`"
+            echo "- Release type: \`canary\`"
+            echo "- Ref: \`main\`"
+          } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/test-publish-py-sdk.yaml

@@ -1,8 +1,6 @@
-#
 # This workflow is used to publish the Python SDK to TestPyPI.
 # It mirrors the main release workflow where practical, but it does not create git tags
 # or GitHub Releases.
-#
 
 name: Publish Python SDK to TestPyPI
 
@@ -14,6 +12,14 @@ on:
         required: true
         type: string
         default: "main"
+      release_type:
+        description: "Release type to publish to TestPyPI"
+        required: true
+        type: choice
+        options:
+          - prerelease
+          - canary
+        default: prerelease
       dry_run:
         description: "Validate and build without publishing to TestPyPI"
         required: true
@@ -27,6 +33,9 @@ jobs:
     outputs:
       commit_sha: ${{ steps.validate.outputs.commit_sha }}
       dry_run: ${{ steps.validate.outputs.dry_run }}
+      release_type: ${{ steps.validate.outputs.release_type }}
+      target_branch: ${{ steps.validate.outputs.target_branch }}
+      version: ${{ steps.validate.outputs.version }}
     steps:
       - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
         with:
@@ -37,26 +46,44 @@ jobs:
         with:
           cache: true
           experimental: true
-      - name: Resolve commit
+      - name: Resolve test release metadata
         id: validate
         run: |
-          echo "commit_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+          RELEASE_TYPE="${{ github.event.inputs.release_type }}"
+          if [ "$RELEASE_TYPE" = "canary" ]; then
+            mise exec -- python py/scripts/test-publish.py prepare canary \
+              "${{ github.event.inputs.ref }}" \
+              --run-number "${{ github.run_number }}" \
+              --github-output "$GITHUB_OUTPUT"
+          else
+            mise exec -- python py/scripts/test-publish.py prepare prerelease \
+              --run-number "${{ github.run_number }}" \
+              --github-output "$GITHUB_OUTPUT"
+          fi
           echo "dry_run=${{ github.event.inputs.dry_run }}" >> "$GITHUB_OUTPUT"
 
   build-and-publish:
     needs: validate
     runs-on: ubuntu-latest
     timeout-minutes: 20
     permissions:
+      actions: read
       id-token: write # Required for PyPI trusted publishing
 
     outputs:
-      version: ${{ steps.get_version.outputs.version }}
+      commit_sha: ${{ steps.publish_status.outputs.commit_sha }}
+      package_name: ${{ steps.publish_status.outputs.package_name }}
+      published: ${{ steps.publish_status.outputs.published }}
+      reason: ${{ steps.publish_status.outputs.reason }}
+      version: ${{ steps.publish_status.outputs.version }}
 
     env:
       COMMIT_SHA: ${{ needs.validate.outputs.commit_sha }}
       DRY_RUN: ${{ needs.validate.outputs.dry_run }}
       PYPI_REPO: testpypi
+      RELEASE_TYPE: ${{ needs.validate.outputs.release_type }}
+      TARGET_BRANCH: ${{ needs.validate.outputs.target_branch }}
+      VERSION: ${{ needs.validate.outputs.version }}
 
     steps:
       - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
@@ -68,55 +95,213 @@ jobs:
         with:
           cache: true
           experimental: true
+      - name: Check whether a new canary is needed
+        if: env.RELEASE_TYPE == 'canary'
+        id: should_publish
+        run: |
+          mise exec -- python py/scripts/test-publish.py check-canary --github-output "$GITHUB_OUTPUT"
+      - name: Check Python CI status
+        if: env.RELEASE_TYPE == 'canary' && steps.should_publish.outputs.should_publish == 'true'
+        id: ci_status
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
+        env:
+          TARGET_BRANCH: ${{ env.TARGET_BRANCH }}
+        with:
+          script: |
+            const { owner, repo } = context.repo;
+            const response = await github.rest.actions.listWorkflowRuns({
+              owner,
+              repo,
+              workflow_id: "py.yaml",
+              branch: process.env.TARGET_BRANCH,
+              status: "completed",
+              per_page: 1,
+            });
+
+            const run = response.data.workflow_runs[0];
+            if (!run) {
+              core.setOutput("should_publish", "false");
+              core.setOutput("reason", `No completed py.yaml run found on ${process.env.TARGET_BRANCH}.`);
+              return;
+            }
+
+            if (run.conclusion !== "success") {
+              core.setOutput("should_publish", "false");
+              core.setOutput(
+                "reason",
+                `Latest completed py.yaml run on ${process.env.TARGET_BRANCH} concluded with ${run.conclusion} (${run.html_url}).`,
+              );
+              return;
+            }
+
+            core.setOutput("should_publish", "true");
+            core.setOutput(
+              "reason",
+              `Latest completed py.yaml run on ${process.env.TARGET_BRANCH} succeeded (${run.html_url}).`,
+            );
       - name: Build and verify
+        if: env.RELEASE_TYPE != 'canary' || (steps.should_publish.outputs.should_publish == 'true' && steps.ci_status.outputs.should_publish == 'true')
+        env:
+          BRAINTRUST_RELEASE_CHANNEL: ${{ env.RELEASE_TYPE }}
+          BRAINTRUST_VERSION_OVERRIDE: ${{ env.VERSION }}
         run: |
           mise exec -- make -C py install-dev verify-build
       - name: Upload build artifacts
+        if: env.RELEASE_TYPE != 'canary' || (steps.should_publish.outputs.should_publish == 'true' && steps.ci_status.outputs.should_publish == 'true')
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: python-sdk-testpypi-dist
           path: py/dist/
           retention-days: 5
-      - name: Get version from built wheel
-        id: get_version
-        run: |
-          WHEEL=$(ls py/dist/*.whl | head -n 1)
-          VERSION=$(echo "$WHEEL" | sed -n 's/.*braintrust-\([^-]*\)-.*/\1/p')
-          echo "version=$VERSION" >> $GITHUB_OUTPUT
       - name: Publish to TestPyPI
-        if: env.DRY_RUN != 'true'
+        if: env.DRY_RUN != 'true' && (env.RELEASE_TYPE != 'canary' || (steps.should_publish.outputs.should_publish == 'true' && steps.ci_status.outputs.should_publish == 'true'))
         uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1
         with:
           repository-url: https://test.pypi.org/legacy/
           packages-dir: py/dist/
       - name: Summarize dry run
         if: env.DRY_RUN == 'true'
         run: |
-          echo "Dry run completed for TestPyPI build from $COMMIT_SHA"
+          {
+            echo "## Python SDK TestPyPI Dry Run Completed"
+            echo
+            echo "- Release type: \`${RELEASE_TYPE}\`"
+            echo "- Version: \`${VERSION}\`"
+            echo "- Commit: \`${COMMIT_SHA}\`"
+          } >> "$GITHUB_STEP_SUMMARY"
+      - name: Summarize skipped canary publish
+        if: env.RELEASE_TYPE == 'canary' && env.DRY_RUN != 'true' && (steps.should_publish.outputs.should_publish != 'true' || steps.ci_status.outputs.should_publish != 'true')
+        env:
+          SHOULD_PUBLISH_REASON: ${{ steps.should_publish.outputs.reason }}
+          CI_REASON: ${{ steps.ci_status.outputs.reason }}
+        run: |
+          set -euo pipefail
+          REASON="${CI_REASON:-$SHOULD_PUBLISH_REASON}"
+          {
+            echo "## Python SDK Canary Publish Skipped"
+            echo
+            echo "$REASON"
+          } >> "$GITHUB_STEP_SUMMARY"
+      - name: Set publish status outputs
+        id: publish_status
+        if: always()
+        env:
+          SHOULD_PUBLISH: ${{ steps.should_publish.outputs.should_publish }}
+          CI_SHOULD_PUBLISH: ${{ steps.ci_status.outputs.should_publish }}
+          SHOULD_PUBLISH_REASON: ${{ steps.should_publish.outputs.reason }}
+          CI_REASON: ${{ steps.ci_status.outputs.reason }}
+        run: |
+          set -euo pipefail
+          echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
+          echo "package_name=braintrust" >> "$GITHUB_OUTPUT"
+          echo "commit_sha=$(git rev-parse --short=7 HEAD)" >> "$GITHUB_OUTPUT"
+
+          if [ "${DRY_RUN}" = "true" ]; then
+            echo "published=false" >> "$GITHUB_OUTPUT"
+            echo "reason=" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          if [ "${RELEASE_TYPE}" != "canary" ]; then
+            echo "published=true" >> "$GITHUB_OUTPUT"
+            echo "reason=" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          if [ "${SHOULD_PUBLISH}" = "true" ] && [ "${CI_SHOULD_PUBLISH}" = "true" ]; then
+            echo "published=true" >> "$GITHUB_OUTPUT"
+            echo "reason=" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          REASON="${CI_REASON:-$SHOULD_PUBLISH_REASON}"
+          echo "published=false" >> "$GITHUB_OUTPUT"
+          echo "reason=${REASON}" >> "$GITHUB_OUTPUT"
 
   notify-success:
     needs: [validate, build-and-publish]
-    if: always() && needs.build-and-publish.result == 'success'
+    if: always() && needs.build-and-publish.result == 'success' && (needs.validate.outputs.dry_run == 'true' || needs.build-and-publish.outputs.published == 'true')
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Post to Slack on dry run success
+        if: needs.validate.outputs.dry_run == 'true'
+        uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
+        with:
+          method: chat.postMessage
+          token: ${{ secrets.SLACK_BOT_TOKEN }}
+          payload: |
+            channel: C0ABHT0SWA2
+            text: "🧪 Python SDK TestPyPI dry run succeeded"
+            blocks:
+              - type: "header"
+                text:
+                  type: "plain_text"
+                  text: "🧪 Python SDK TestPyPI Dry Run Succeeded"
+              - type: "section"
+                text:
+                  type: "mrkdwn"
+                  text: "${{ format('*Mode:* dry run\n*Release type:* {0}\n*Version:* {1}\n*Ref:* {2}\n*Commit:* {3}\n\n<{4}/{5}/actions/runs/{6}|View Run>', needs.validate.outputs.release_type, needs.validate.outputs.version, github.event.inputs.ref, needs.validate.outputs.commit_sha, github.server_url, github.repository, github.run_id) }}"
+      - name: Post to Slack on prerelease success
+        if: needs.validate.outputs.dry_run != 'true' && needs.validate.outputs.release_type == 'prerelease'
+        uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
+        with:
+          method: chat.postMessage
+          token: ${{ secrets.SLACK_BOT_TOKEN }}
+          payload: |
+            channel: C0ABHT0SWA2
+            text: "🧪 Python SDK prerelease v${{ needs.build-and-publish.outputs.version }} published to TestPyPI"
+            blocks:
+              - type: "header"
+                text:
+                  type: "plain_text"
+                  text: "🧪 Python SDK Pre-release Published"
+              - type: "section"
+                text:
+                  type: "mrkdwn"
+                  text: "${{ format('*Version:* {0}\n*Ref:* {1}\n*Install:* `pip install -i https://test.pypi.org/simple/ braintrust=={0}`\n*Package:* <https://test.pypi.org/project/braintrust/|braintrust (TestPyPI)>\n\n<{2}/{3}/actions/runs/{4}|View Run>', needs.build-and-publish.outputs.version, github.event.inputs.ref, github.server_url, github.repository, github.run_id) }}"
+      - name: Post to Slack on canary success
+        if: needs.validate.outputs.dry_run != 'true' && needs.validate.outputs.release_type == 'canary'
+        uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
+        with:
+          method: chat.postMessage
+          token: ${{ secrets.SLACK_BOT_TOKEN }}
+          payload: |
+            channel: C0ABHT0SWA2
+            text: "🧪 Python SDK canary ${{ needs.build-and-publish.outputs.version }} published to TestPyPI"
+            blocks:
+              - type: "header"
+                text:
+                  type: "plain_text"
+                  text: "🧪 Python SDK Canary Published"
+              - type: "section"
+                text:
+                  type: "mrkdwn"
+                  text: "${{ format('*Version:* {0}\n*Branch:* `{1}`\n*Commit:* `{2}`\n*Install:* `pip install -i https://test.pypi.org/simple/ braintrust=={0}`\n*Package:* <https://test.pypi.org/project/braintrust/|braintrust (TestPyPI)>\n\n<{3}/{4}/actions/runs/{5}|View Run>', needs.build-and-publish.outputs.version, needs.validate.outputs.target_branch, needs.build-and-publish.outputs.commit_sha, github.server_url, github.repository, github.run_id) }}"
+
+  notify-skipped:
+    needs: [validate, build-and-publish]
+    if: always() && needs.validate.outputs.release_type == 'canary' && needs.validate.outputs.dry_run != 'true' && needs.build-and-publish.result == 'success' && needs.build-and-publish.outputs.published != 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
-      - name: Post to Slack on success
+      - name: Post to Slack on intentional skip
         uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
         with:
           method: chat.postMessage
           token: ${{ secrets.SLACK_BOT_TOKEN }}
           payload: |
             channel: C0ABHT0SWA2
-            text: "${{ needs.validate.outputs.dry_run == 'true' && '🧪 Python SDK TestPyPI dry run succeeded' || format('🧪 Python SDK prerelease v{0} published to TestPyPI', needs.build-and-publish.outputs.version) }}"
+            text: "⏭️ Python SDK canary publish skipped: ${{ needs.build-and-publish.outputs.reason }}"
             blocks:
               - type: "header"
                 text:
                   type: "plain_text"
-                  text: "${{ needs.validate.outputs.dry_run == 'true' && '🧪 Python SDK TestPyPI Dry Run Succeeded' || '🧪 Python SDK Pre-release Published' }}"
+                  text: "⏭️ Python SDK Canary Publish Skipped"
               - type: "section"
                 text:
                   type: "mrkdwn"
-                  text: "${{ needs.validate.outputs.dry_run == 'true' && format('*Mode:* dry run\n*Ref:* {0}\n*Commit:* {1}\n\n<{2}/{3}/actions/runs/{4}|View Run>', github.event.inputs.ref, needs.validate.outputs.commit_sha, github.server_url, github.repository, github.run_id) || format('*Version:* {0}\n*Ref:* {1}\n*Install:* `pip install -i https://test.pypi.org/simple/ braintrust=={0}`\n*Package:* <https://test.pypi.org/project/braintrust/|braintrust (TestPyPI)>\n\n<{2}/{3}/actions/runs/{4}|View Run>', needs.build-and-publish.outputs.version, github.event.inputs.ref, github.server_url, github.repository, github.run_id) }}"
+                  text: "${{ format('*Branch:* `{0}`\n*Commit:* `{1}`\n*Reason:* {2}\n\n<{3}/{4}/actions/runs/{5}|View Run>', needs.validate.outputs.target_branch, needs.build-and-publish.outputs.commit_sha, needs.build-and-publish.outputs.reason, github.server_url, github.repository, github.run_id) }}"
 
   notify-failure:
     needs: [validate, build-and-publish]

AGENTS.md

@@ -133,7 +133,7 @@ make build
 
 Important caveat:
 
-- `py/scripts/template-version.sh` rewrites `py/src/braintrust/version.py` during build.
+- `py/scripts/template-version.py` rewrites `py/src/braintrust/version.py` during build.
 - `py/Makefile` restores that file afterward with `git checkout`.
 
 Avoid editing `py/src/braintrust/version.py` while also running build commands.