From d467dd0105861b592a15c9fff81935d947210186 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 9 Jul 2026 10:33:28 +0200 Subject: [PATCH 1/6] ci: publish to public PyPI on release + TestPyPI dry-run workflow Add public PyPI publishing alongside the existing private Azure Artifacts feed, via OIDC Trusted Publishing (no stored token): - release.yml: expose the release job's `version` as an output and upload the built wheel+sdist as an artifact; add an environment-gated `publish-pypi` job (`needs: release`, `id-token: write`, GitHub-hosted runner) that downloads that artifact and publishes to pypi.org. The Azure feed and GHCR image steps are unchanged. - publish-testpypi.yml: new manually-dispatched dry run that stamps a throwaway `.dev` version and publishes to test.pypi.org, so the build + OIDC path can be validated before a real release. All third-party actions are pinned to full commit SHAs (Dependabot's github-actions ecosystem keeps them current). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/publish-testpypi.yml | 92 ++++++++++++++++++++++++++ .github/workflows/release.yml | 46 +++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 .github/workflows/publish-testpypi.yml diff --git a/.github/workflows/publish-testpypi.yml b/.github/workflows/publish-testpypi.yml new file mode 100644 index 00000000..f219e8b3 --- /dev/null +++ b/.github/workflows/publish-testpypi.yml @@ -0,0 +1,92 @@ +name: Publish to TestPyPI (dry run) + +# Manually-triggered dry run of the public-PyPI publish path against +# https://test.pypi.org. Use this to validate the build + OIDC Trusted +# Publishing flow BEFORE cutting a real release (release.yml -> pypi.org). +# +# This does NOT bump the version, tag, or touch main / the Azure feed. It +# stamps a throwaway PEP 440 dev version (`.dev`) so repeated +# runs never collide with an already-uploaded TestPyPI file, builds, and +# publishes via OIDC (no stored token). +# +# One-time setup (mirrors the real PyPI publisher, on the TEST instance): +# test.pypi.org -> Account settings -> Publishing -> add a pending publisher: +# Project name: coder-eval +# Owner: UiPath +# Repository: coder_eval +# Workflow name: publish-testpypi.yml +# Environment: testpypi +# +# Install the dry-run build to smoke-test it: +# uv pip install --index-url https://test.pypi.org/simple/ \ +# --extra-index-url https://pypi.org/simple/ coder-eval + +on: + workflow_dispatch: + +concurrency: + group: publish-testpypi + cancel-in-progress: false + +permissions: + contents: read + +jobs: + publish-testpypi: + name: Build and publish to TestPyPI + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: + name: testpypi + url: https://test.pypi.org/project/coder-eval/ + permissions: + # OIDC token minting for Trusted Publishing; no long-lived credentials. + id-token: write + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Python 3.13 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.13" + + - name: Install uv + uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 + with: + enable-cache: true + + - name: Stamp a throwaway dev version + env: + RUN_NUMBER: ${{ github.run_number }} + run: | + set -euo pipefail + CURRENT_VERSION=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])") + # PEP 440 dev release: sorts before the base release, unique per run. + DEV_VERSION="${CURRENT_VERSION}.dev${RUN_NUMBER}" + python3 - < no version bumped => skip PyPI publish). + version: ${{ steps.release.outputs.version }} env: # uipath-ubuntu-latest runners enforce a minimum package-age safe-chain # check on uv installs; match the exclusions used by pr-checks.yml so @@ -118,6 +122,17 @@ jobs: if: steps.release.outputs.version != '' run: uv build + # Hand the exact built artifacts to the publish-pypi job. Publishing to + # public PyPI runs in its own environment-gated job (OIDC), so it must + # consume these files rather than rebuild them. + - name: Upload dist for PyPI publish + if: steps.release.outputs.version != '' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: release-dist + path: dist/ + if-no-files-found: error + - name: Publish to coder_eval feed if: steps.release.outputs.version != '' env: @@ -170,3 +185,34 @@ jobs: # Read the shared buildcache docker-publish.yml writes; don't write it # back from here, to avoid two concurrent writers racing the cache tag. cache-from: type=registry,ref=ghcr.io/${{ steps.img.outputs.owner_lc }}/coder-eval-agent:buildcache + + # Publish the SAME wheel+sdist to public PyPI (pkgs live alongside the private + # Azure Artifacts feed, which the release job above still populates). This runs + # as its own job so OIDC Trusted Publishing is scoped to a dedicated, + # environment-gated context on GitHub-hosted runners -- no PyPI token/secret is + # stored. Gated on the release job having actually cut a version. + publish-pypi: + name: Publish to PyPI + needs: release + if: needs.release.outputs.version != '' + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: + name: pypi + url: https://pypi.org/project/coder-eval/${{ needs.release.outputs.version }}/ + permissions: + # OIDC token minting for Trusted Publishing; no long-lived credentials. + id-token: write + steps: + - name: Download built dist + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: release-dist + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + with: + # Trusted Publisher is configured on pypi.org for this repo + + # workflow (release.yml) + environment (pypi); no password needed. + packages-dir: dist/ From b69e4cfc693de67b126d2d0d9aeb70a968a1de2b Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 9 Jul 2026 11:04:20 +0200 Subject: [PATCH 2/6] ci: bump claude-code-action v1.0.134 -> v1.0.169 The pinned v1.0.134 drops thinking blocks across tool_use turns, so Bedrock rejects the reviewer request with `Expected thinking or redacted_thinking, but found tool_use`. Bump to the latest release to pick up the fix; SHA-pinned, Dependabot keeps it current. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/claude-pr-review.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/claude-pr-review.yml b/.github/workflows/claude-pr-review.yml index 839530aa..45c47087 100644 --- a/.github/workflows/claude-pr-review.yml +++ b/.github/workflows/claude-pr-review.yml @@ -88,7 +88,7 @@ jobs: # path; this bearer-token path skips that infra and is the same one # the Claude CLI uses directly (exercised end-to-end by the # live-tests BedrockRoute step). - uses: anthropics/claude-code-action@09f876c77e9d2d627cdf3e2f83d4d7b75106807e # v1.0.134 + uses: anthropics/claude-code-action@37b464ce72700f7b2c5ff8d2db7fa7b15df792f5 # v1.0.169 env: AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} AWS_REGION: ${{ secrets.AWS_REGION }} From 0b19b26a81694fc1a67052e0367b617995b46f4c Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 9 Jul 2026 11:38:29 +0200 Subject: [PATCH 3/6] ci: run release job on GitHub-hosted ubuntu-latest The release (bump + build + Azure feed + GHCR) job ran on the self-hosted `uipath-ubuntu-latest` pool, so cutting a release was blocked whenever that pool was unavailable -- and publish-pypi (needs: release) with it. Move it to GitHub-hosted ubuntu-latest so PyPI releases are independent of the self-hosted runners. semantic-release, uv, twine, and docker buildx -> GHCR all run there. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/release.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1a9e7f6b..84d9c431 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -41,16 +41,20 @@ permissions: jobs: release: name: Bump version and publish - runs-on: uipath-ubuntu-latest + # GitHub-hosted so cutting a release does not depend on the self-hosted + # `uipath-ubuntu-latest` pool. semantic-release, uv, twine, and the docker + # buildx -> GHCR push all run fine here. + runs-on: ubuntu-latest timeout-minutes: 15 outputs: # Exposed so the downstream publish-pypi job can gate on a real release # having been cut (empty => no version bumped => skip PyPI publish). version: ${{ steps.release.outputs.version }} env: - # uipath-ubuntu-latest runners enforce a minimum package-age safe-chain - # check on uv installs; match the exclusions used by pr-checks.yml so - # parity is maintained if/when codex deps flow through the release path. + # The self-hosted `uipath-ubuntu-latest` runners enforce a minimum + # package-age safe-chain check on uv installs; on GitHub-hosted runners + # this is a no-op. Kept (matching pr-checks.yml) so parity is preserved + # if the job ever moves back to the self-hosted pool. SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: "openai-codex-cli-bin,openai-codex" steps: From ba81e262265f8f8b9dc2f85e14b2b28e22891660 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 9 Jul 2026 11:46:27 +0200 Subject: [PATCH 4/6] ci: fix .venv cache 127 and Codex sqlite migration race Two pre-existing CI infra bugs: 1. `.venv` cache exit 127 -- the Linux cache blocks cached `.venv` keyed only on major.minor (`py3.13`), so a hosted-Python patch bump left `.venv/bin/python` a dangling symlink and every `.venv/bin/*` failed with "cannot execute: required file not found". Drop `.venv` from the three Linux cache paths (Quality Gate / E2E / Live); `uv sync --frozen` rehydrates it from the cached wheels, matching what the Windows job already does. 2. Codex live test `duplicate column name: thread_id` -- addopts pins `-n auto`, so xdist workers shared one ~/.codex and raced the Codex SQLite state migration. Run the Codex live suite serially with `-n0`. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/pr-checks.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index cb99e268..eb534722 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -51,10 +51,14 @@ jobs: - name: Cache dependencies uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: + # ``.venv`` is deliberately NOT cached: its interpreter symlinks + # point at the exact hosted-Python patch, so a patch bump (e.g. + # 3.13.13 -> 3.13.14) leaves a dangling ``.venv/bin/python`` and + # every ``.venv/bin/*`` fails with exit 127 (cannot execute). + # ``uv sync --frozen`` rehydrates ``.venv`` from the cached wheels. path: | ~/.cache/uv ~/.cache/pip - .venv .pytest_cache .ruff_cache key: ${{ runner.os }}-py3.13-${{ hashFiles('pyproject.toml', 'uv.lock') }} @@ -379,10 +383,12 @@ jobs: - name: Cache dependencies uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: + # ``.venv`` excluded: a hosted-Python patch bump orphans its + # interpreter symlink (dangling ``.venv/bin/python`` -> exit 127). + # ``uv sync --frozen`` rehydrates it from the cached wheels. path: | ~/.cache/uv ~/.cache/pip - .venv key: ${{ runner.os }}-py3.13-e2e-${{ hashFiles('pyproject.toml', 'uv.lock') }} restore-keys: | ${{ runner.os }}-py3.13-e2e- @@ -544,10 +550,12 @@ jobs: - name: Cache dependencies uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: + # ``.venv`` excluded: a hosted-Python patch bump orphans its + # interpreter symlink (dangling ``.venv/bin/python`` -> exit 127). + # ``uv sync --frozen`` rehydrates it from the cached wheels. path: | ~/.cache/uv ~/.cache/pip - .venv key: ${{ runner.os }}-py3.13-live-${{ hashFiles('pyproject.toml', 'uv.lock') }} restore-keys: | ${{ runner.os }}-py3.13-live- @@ -714,8 +722,12 @@ jobs: - name: Run Codex live tests run: | mkdir -p tmp + # Run serially: `-n0` overrides the global `-n auto` (addopts). + # Parallel xdist workers share ~/.codex and race the Codex SQLite + # state migration (`duplicate column name: thread_id`); serial init + # migrates the fresh DB exactly once. .venv/bin/pytest tests/test_codex_agent_live.py \ - -m live -v --tb=short --strict-markers -ra \ + -m live -n0 -v --tb=short --strict-markers -ra \ --junit-xml=tmp/junit-codex-live.xml - name: Assert Codex live tests actually ran (not silently skipped) From 28e3ac40818b3809102f38d10e78deb0e00f0499 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 9 Jul 2026 11:52:54 +0200 Subject: [PATCH 5/6] ci: move all jobs off self-hosted uipath-ubuntu-latest to ubuntu-latest The self-hosted `uipath-ubuntu-latest` pool is unavailable to this repo, so every job pinned to it (Quality Gate, No-Extra Install, E2E Smoke, CodeQL, docker-publish, publish-dev) sat queued indefinitely. Move them all to GitHub-hosted `ubuntu-latest` so CI runs without depending on the pool. Note: this drops the self-hosted `SAFE_CHAIN_MINIMUM_PACKAGE_AGE` supply-chain check on uv installs (a no-op on GitHub-hosted); the env is kept so parity returns if a job ever moves back. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/codeql.yml | 2 +- .github/workflows/docker-publish.yml | 2 +- .github/workflows/pr-checks.yml | 6 +++--- .github/workflows/publish-dev.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0d3ac884..02ead00b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -16,7 +16,7 @@ permissions: jobs: analyze: name: Analyze Python code - runs-on: uipath-ubuntu-latest + runs-on: ubuntu-latest timeout-minutes: 15 permissions: security-events: write diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 1b1d9d4a..226b70c5 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -41,7 +41,7 @@ env: jobs: publish: name: Build and push to GHCR - runs-on: uipath-ubuntu-latest + runs-on: ubuntu-latest # Skip semantic-release's own commit so we don't double-build on the # version-bump push. if: "!contains(github.event.head_commit.message, 'chore(release):')" diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index eb534722..28578b09 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -30,7 +30,7 @@ env: jobs: quality-gate: name: Quality Gate (Format, Lint, Type, Test, Security) - runs-on: uipath-ubuntu-latest + runs-on: ubuntu-latest timeout-minutes: 10 # Shared env for all steps: safe-chain min-age exclusions and a dummy @@ -163,7 +163,7 @@ jobs: # registry validates, and the uipath-specific code paths fail with a # clear hint instead of an import error. name: No-Extra Install (uipath optional) - runs-on: uipath-ubuntu-latest + runs-on: ubuntu-latest timeout-minutes: 5 steps: - name: Checkout code @@ -326,7 +326,7 @@ jobs: e2e-smoke: name: E2E Smoke Tests (Real API) - runs-on: uipath-ubuntu-latest + runs-on: ubuntu-latest timeout-minutes: 10 # Skip on fork PRs where secrets aren't available if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository diff --git a/.github/workflows/publish-dev.yml b/.github/workflows/publish-dev.yml index 1f385807..a79c096a 100644 --- a/.github/workflows/publish-dev.yml +++ b/.github/workflows/publish-dev.yml @@ -28,7 +28,7 @@ permissions: jobs: publish-dev: name: Publish dev wheel for PR - runs-on: uipath-ubuntu-latest + runs-on: ubuntu-latest if: contains(github.event.pull_request.labels.*.name, 'build:dev') timeout-minutes: 15 env: From b5c2871c64566ef2d482ee17ff712b12a54cb165 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 9 Jul 2026 12:01:52 +0200 Subject: [PATCH 6/6] =?UTF-8?q?ci:=20address=20review=20=E2=80=94=20SHA-pi?= =?UTF-8?q?n=20release=20actions=20+=20fail-loud=20dev-version=20stamp?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two medium findings from the PR review: - [security] The wheel/sdist now published to PUBLIC PyPI is built in the release job, which pulled actions via mutable tags (setup-uv@v4, docker/{setup-buildx,login,build-push}@vN). A repointed tag / upstream takeover would reach every `pip install coder-eval` user, in a job that also holds RELEASE_APP_PRIVATE_KEY and the Azure PAT. Pin all four to full commit SHAs (setup-uv now matches publish-testpypi.yml). - [resilience] The TestPyPI dev-version stamp used re.sub, which returns the text unchanged (rc=0) on a version-line format drift, silently publishing the base version -> TestPyPI collision masked by skip-existing, so the dry run passes green having validated nothing. Switch to re.subn and SystemExit when the match count != 1. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/publish-testpypi.yml | 8 +++++++- .github/workflows/release.yml | 8 ++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish-testpypi.yml b/.github/workflows/publish-testpypi.yml index f219e8b3..0a864259 100644 --- a/.github/workflows/publish-testpypi.yml +++ b/.github/workflows/publish-testpypi.yml @@ -73,7 +73,13 @@ jobs: ): p = pathlib.Path(path) key = "version" if path.endswith("toml") else "__version__" - p.write_text(re.sub(pat, f'{key} = "${DEV_VERSION}"', p.read_text(), count=1)) + # re.subn (not re.sub) so a version-line format drift aborts loudly + # instead of silently no-op'ing (rc=0) and publishing a stale/base + # version that then collides on TestPyPI (masked by skip-existing). + new, n = re.subn(pat, f'{key} = "${DEV_VERSION}"', p.read_text(), count=1) + if n != 1: + raise SystemExit(f"version pattern did not match {path} (matched {n}); refusing to publish a stale/colliding artifact") + p.write_text(new) PY echo "Stamped dev version: ${DEV_VERSION}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 84d9c431..b14e10ab 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -79,7 +79,7 @@ jobs: python-version: "3.13" - name: Install uv - uses: astral-sh/setup-uv@v4 + uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 with: enable-cache: true @@ -161,11 +161,11 @@ jobs: - name: Set up Docker Buildx if: steps.release.outputs.version != '' - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Log in to GHCR if: steps.release.outputs.version != '' - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -173,7 +173,7 @@ jobs: - name: Build and push versioned agent image if: steps.release.outputs.version != '' - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: . file: docker/Dockerfile