diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d29c06e6f05..322aad46f45 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,12 +1,6 @@ # Core crate ownership. /codex-rs/core/ @openai/codex-core-agent-team /codex-rs/ext/extension-api/ @openai/codex-core-agent-team -/codex-rs/prompts/ @openai/codex-core-agent-team - -# Keep macOS AKV signing changes reviewed by Codex maintainers. -/.github/actions/setup-akv-pkcs11-codesigning/ @openai/codex-core-agent-team -/.github/scripts/macos-signing/ @openai/codex-core-agent-team -/.github/workflows/rust-release.yml @openai/codex-core-agent-team # Keep ownership changes reviewed by the same team. /.github/CODEOWNERS @openai/codex-core-agent-team diff --git a/.github/actions/macos-code-sign/action.yml b/.github/actions/macos-code-sign/action.yml new file mode 100644 index 00000000000..0e19fa11d0a --- /dev/null +++ b/.github/actions/macos-code-sign/action.yml @@ -0,0 +1,259 @@ +name: macos-code-sign +description: Configure, sign, notarize, and clean up macOS code signing artifacts. +inputs: + target: + description: Rust compilation target triple (e.g. aarch64-apple-darwin). + required: true + binaries: + description: Space-delimited binary basenames to sign and notarize. + default: "codex codex-responses-api-proxy" + sign-binaries: + description: Whether to sign and notarize the macOS binaries. + required: false + default: "true" + sign-dmg: + description: Whether to sign and notarize the macOS dmg. + required: false + default: "true" + apple-certificate: + description: Base64-encoded Apple signing certificate (P12). + required: true + apple-certificate-password: + description: Password for the signing certificate. + required: true + apple-notarization-key-p8: + description: Base64-encoded Apple notarization key (P8). + required: true + apple-notarization-key-id: + description: Apple notarization key ID. + required: true + apple-notarization-issuer-id: + description: Apple notarization issuer ID. + required: true +runs: + using: composite + steps: + - name: Configure Apple code signing + shell: bash + env: + KEYCHAIN_PASSWORD: actions + APPLE_CERTIFICATE: ${{ inputs.apple-certificate }} + APPLE_CERTIFICATE_PASSWORD: ${{ inputs.apple-certificate-password }} + run: | + set -euo pipefail + + if [[ -z "${APPLE_CERTIFICATE:-}" ]]; then + echo "APPLE_CERTIFICATE is required for macOS signing" + exit 1 + fi + + if [[ -z "${APPLE_CERTIFICATE_PASSWORD:-}" ]]; then + echo "APPLE_CERTIFICATE_PASSWORD is required for macOS signing" + exit 1 + fi + + cert_path="${RUNNER_TEMP}/apple_signing_certificate.p12" + echo "$APPLE_CERTIFICATE" | base64 -d > "$cert_path" + + keychain_path="${RUNNER_TEMP}/codex-signing.keychain-db" + security create-keychain -p "$KEYCHAIN_PASSWORD" "$keychain_path" + security set-keychain-settings -lut 21600 "$keychain_path" + security unlock-keychain -p "$KEYCHAIN_PASSWORD" "$keychain_path" + + keychain_args=() + cleanup_keychain() { + if ((${#keychain_args[@]} > 0)); then + security list-keychains -s "${keychain_args[@]}" || true + security default-keychain -s "${keychain_args[0]}" || true + else + security list-keychains -s || true + fi + if [[ -f "$keychain_path" ]]; then + security delete-keychain "$keychain_path" || true + fi + } + + while IFS= read -r keychain; do + [[ -n "$keychain" ]] && keychain_args+=("$keychain") + done < <(security list-keychains | sed 's/^[[:space:]]*//;s/[[:space:]]*$//;s/"//g') + + if ((${#keychain_args[@]} > 0)); then + security list-keychains -s "$keychain_path" "${keychain_args[@]}" + else + security list-keychains -s "$keychain_path" + fi + + security default-keychain -s "$keychain_path" + security import "$cert_path" -k "$keychain_path" -P "$APPLE_CERTIFICATE_PASSWORD" -T /usr/bin/codesign -T /usr/bin/security + security set-key-partition-list -S apple-tool:,apple: -s -k "$KEYCHAIN_PASSWORD" "$keychain_path" > /dev/null + + codesign_hashes=() + while IFS= read -r hash; do + [[ -n "$hash" ]] && codesign_hashes+=("$hash") + done < <(security find-identity -v -p codesigning "$keychain_path" \ + | sed -n 's/.*\([0-9A-F]\{40\}\).*/\1/p' \ + | sort -u) + + if ((${#codesign_hashes[@]} == 0)); then + echo "No signing identities found in $keychain_path" + cleanup_keychain + rm -f "$cert_path" + exit 1 + fi + + if ((${#codesign_hashes[@]} > 1)); then + echo "Multiple signing identities found in $keychain_path:" + printf ' %s\n' "${codesign_hashes[@]}" + cleanup_keychain + rm -f "$cert_path" + exit 1 + fi + + APPLE_CODESIGN_IDENTITY="${codesign_hashes[0]}" + + rm -f "$cert_path" + + echo "APPLE_CODESIGN_IDENTITY=$APPLE_CODESIGN_IDENTITY" >> "$GITHUB_ENV" + echo "APPLE_CODESIGN_KEYCHAIN=$keychain_path" >> "$GITHUB_ENV" + echo "::add-mask::$APPLE_CODESIGN_IDENTITY" + + - name: Sign macOS binaries + if: ${{ inputs.sign-binaries == 'true' }} + shell: bash + env: + TARGET: ${{ inputs.target }} + BINARIES: ${{ inputs.binaries }} + run: | + set -euo pipefail + + if [[ -z "${APPLE_CODESIGN_IDENTITY:-}" ]]; then + echo "APPLE_CODESIGN_IDENTITY is required for macOS signing" + exit 1 + fi + + keychain_args=() + if [[ -n "${APPLE_CODESIGN_KEYCHAIN:-}" && -f "${APPLE_CODESIGN_KEYCHAIN}" ]]; then + keychain_args+=(--keychain "${APPLE_CODESIGN_KEYCHAIN}") + fi + + entitlements_path="$GITHUB_ACTION_PATH/codex.entitlements.plist" + + for binary in ${BINARIES}; do + path="codex-rs/target/${TARGET}/release/${binary}" + codesign --force --options runtime --timestamp --entitlements "$entitlements_path" --sign "$APPLE_CODESIGN_IDENTITY" "${keychain_args[@]}" "$path" + done + + - name: Notarize macOS binaries + if: ${{ inputs.sign-binaries == 'true' }} + shell: bash + env: + TARGET: ${{ inputs.target }} + BINARIES: ${{ inputs.binaries }} + APPLE_NOTARIZATION_KEY_P8: ${{ inputs.apple-notarization-key-p8 }} + APPLE_NOTARIZATION_KEY_ID: ${{ inputs.apple-notarization-key-id }} + APPLE_NOTARIZATION_ISSUER_ID: ${{ inputs.apple-notarization-issuer-id }} + run: | + set -euo pipefail + + for var in APPLE_NOTARIZATION_KEY_P8 APPLE_NOTARIZATION_KEY_ID APPLE_NOTARIZATION_ISSUER_ID; do + if [[ -z "${!var:-}" ]]; then + echo "$var is required for notarization" + exit 1 + fi + done + + notary_key_path="${RUNNER_TEMP}/notarytool.key.p8" + echo "$APPLE_NOTARIZATION_KEY_P8" | base64 -d > "$notary_key_path" + cleanup_notary() { + rm -f "$notary_key_path" + } + trap cleanup_notary EXIT + + source "$GITHUB_ACTION_PATH/notary_helpers.sh" + + notarize_binary() { + local binary="$1" + local source_path="codex-rs/target/${TARGET}/release/${binary}" + local archive_path="${RUNNER_TEMP}/${binary}.zip" + + if [[ ! -f "$source_path" ]]; then + echo "Binary $source_path not found" + exit 1 + fi + + rm -f "$archive_path" + ditto -c -k --keepParent "$source_path" "$archive_path" + + notarize_submission "$binary" "$archive_path" "$notary_key_path" + } + + for binary in ${BINARIES}; do + notarize_binary "${binary}" + done + + - name: Sign and notarize macOS dmg + if: ${{ inputs.sign-dmg == 'true' }} + shell: bash + env: + TARGET: ${{ inputs.target }} + APPLE_NOTARIZATION_KEY_P8: ${{ inputs.apple-notarization-key-p8 }} + APPLE_NOTARIZATION_KEY_ID: ${{ inputs.apple-notarization-key-id }} + APPLE_NOTARIZATION_ISSUER_ID: ${{ inputs.apple-notarization-issuer-id }} + run: | + set -euo pipefail + + for var in APPLE_CODESIGN_IDENTITY APPLE_NOTARIZATION_KEY_P8 APPLE_NOTARIZATION_KEY_ID APPLE_NOTARIZATION_ISSUER_ID; do + if [[ -z "${!var:-}" ]]; then + echo "$var is required" + exit 1 + fi + done + + notary_key_path="${RUNNER_TEMP}/notarytool.key.p8" + echo "$APPLE_NOTARIZATION_KEY_P8" | base64 -d > "$notary_key_path" + cleanup_notary() { + rm -f "$notary_key_path" + } + trap cleanup_notary EXIT + + source "$GITHUB_ACTION_PATH/notary_helpers.sh" + + dmg_name="codex-${TARGET}.dmg" + dmg_path="codex-rs/target/${TARGET}/release/${dmg_name}" + + if [[ ! -f "$dmg_path" ]]; then + echo "dmg $dmg_path not found" + exit 1 + fi + + keychain_args=() + if [[ -n "${APPLE_CODESIGN_KEYCHAIN:-}" && -f "${APPLE_CODESIGN_KEYCHAIN}" ]]; then + keychain_args+=(--keychain "${APPLE_CODESIGN_KEYCHAIN}") + fi + + codesign --force --timestamp --sign "$APPLE_CODESIGN_IDENTITY" "${keychain_args[@]}" "$dmg_path" + notarize_submission "$dmg_name" "$dmg_path" "$notary_key_path" + xcrun stapler staple "$dmg_path" + + - name: Remove signing keychain + if: ${{ always() }} + shell: bash + env: + APPLE_CODESIGN_KEYCHAIN: ${{ env.APPLE_CODESIGN_KEYCHAIN }} + run: | + set -euo pipefail + if [[ -n "${APPLE_CODESIGN_KEYCHAIN:-}" ]]; then + keychain_args=() + while IFS= read -r keychain; do + [[ "$keychain" == "$APPLE_CODESIGN_KEYCHAIN" ]] && continue + [[ -n "$keychain" ]] && keychain_args+=("$keychain") + done < <(security list-keychains | sed 's/^[[:space:]]*//;s/[[:space:]]*$//;s/"//g') + if ((${#keychain_args[@]} > 0)); then + security list-keychains -s "${keychain_args[@]}" + security default-keychain -s "${keychain_args[0]}" + fi + + if [[ -f "$APPLE_CODESIGN_KEYCHAIN" ]]; then + security delete-keychain "$APPLE_CODESIGN_KEYCHAIN" + fi + fi diff --git a/.github/scripts/macos-signing/codex.entitlements.plist b/.github/actions/macos-code-sign/codex.entitlements.plist similarity index 100% rename from .github/scripts/macos-signing/codex.entitlements.plist rename to .github/actions/macos-code-sign/codex.entitlements.plist diff --git a/.github/actions/macos-code-sign/notary_helpers.sh b/.github/actions/macos-code-sign/notary_helpers.sh new file mode 100644 index 00000000000..ad9757fe3cb --- /dev/null +++ b/.github/actions/macos-code-sign/notary_helpers.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +notarize_submission() { + local label="$1" + local path="$2" + local notary_key_path="$3" + + if [[ -z "${APPLE_NOTARIZATION_KEY_ID:-}" || -z "${APPLE_NOTARIZATION_ISSUER_ID:-}" ]]; then + echo "APPLE_NOTARIZATION_KEY_ID and APPLE_NOTARIZATION_ISSUER_ID are required for notarization" + exit 1 + fi + + if [[ -z "$notary_key_path" || ! -f "$notary_key_path" ]]; then + echo "Notary key file $notary_key_path not found" + exit 1 + fi + + if [[ ! -f "$path" ]]; then + echo "Notarization payload $path not found" + exit 1 + fi + + local submission_json + submission_json=$(xcrun notarytool submit "$path" \ + --key "$notary_key_path" \ + --key-id "$APPLE_NOTARIZATION_KEY_ID" \ + --issuer "$APPLE_NOTARIZATION_ISSUER_ID" \ + --output-format json \ + --wait) + + local status submission_id + status=$(printf '%s\n' "$submission_json" | jq -r '.status // "Unknown"') + submission_id=$(printf '%s\n' "$submission_json" | jq -r '.id // ""') + + if [[ -z "$submission_id" ]]; then + echo "Failed to retrieve submission ID for $label" + exit 1 + fi + + echo "::notice title=Notarization::$label submission ${submission_id} completed with status ${status}" + + if [[ "$status" != "Accepted" ]]; then + echo "Notarization failed for ${label} (submission ${submission_id}, status ${status})" + exit 1 + fi +} diff --git a/.github/actions/setup-akv-pkcs11-codesigning/action.yaml b/.github/actions/setup-akv-pkcs11-codesigning/action.yaml deleted file mode 100644 index 4d79cab42ef..00000000000 --- a/.github/actions/setup-akv-pkcs11-codesigning/action.yaml +++ /dev/null @@ -1,349 +0,0 @@ -name: Set up AKV PKCS11 code signing -description: Download prebuilt rcodesign and Azure Key Vault PKCS11 provider artifacts, then export macOS signing environment. - -inputs: - setup-mode: - description: signing configures Azure and exports signing env vars; tools-only only downloads signing tools. - required: false - default: signing - rcodesign-blob-uri: - description: Azure Blob URI for the prebuilt Linux/amd64 rcodesign binary. - required: true - rcodesign-sha256: - description: Expected SHA-256 digest for the prebuilt rcodesign binary. - required: true - akv-pkcs11-library-blob-uri: - description: Azure Blob URI for the prebuilt Linux/amd64 AKV PKCS11 provider library. - required: true - akv-pkcs11-library-sha256: - description: Expected SHA-256 digest for the prebuilt AKV PKCS11 provider library. - required: true - azure-client-id: - description: GitHub OIDC client ID for the Azure signer application. - required: true - azure-tenant-id: - description: Azure tenant ID for the signer application. - required: true - azure-subscription-id: - description: Azure subscription ID that owns the signing vault. - required: true - key-vault-name: - description: Azure Key Vault name containing the certificate-backed signing key. - required: true - key-name: - description: Key Vault certificate/key name used as the PKCS11 key label. - required: true - key-version: - description: Optional Key Vault key version to pin while signing. - required: false - default: "" - certificate-sha256: - description: Optional expected SHA-256 fingerprint for the downloaded public certificate. - required: false - default: "" - -outputs: - pkcs11-library: - description: Path to the downloaded AKV PKCS11 provider library. - value: ${{ steps.paths.outputs.pkcs11_library }} - signing-certificate-pem: - description: Path to the downloaded public signing certificate. - value: ${{ steps.paths.outputs.signing_certificate_pem }} - rcodesign: - description: Path to the downloaded rcodesign binary. - value: ${{ steps.paths.outputs.rcodesign }} - -runs: - using: composite - steps: - - name: Validate pinned signing artifacts - shell: bash - env: - SETUP_MODE: ${{ inputs.setup-mode }} - RCODESIGN_BLOB_URI: ${{ inputs.rcodesign-blob-uri }} - RCODESIGN_SHA256: ${{ inputs.rcodesign-sha256 }} - AKV_PKCS11_LIBRARY_BLOB_URI: ${{ inputs.akv-pkcs11-library-blob-uri }} - AKV_PKCS11_LIBRARY_SHA256: ${{ inputs.akv-pkcs11-library-sha256 }} - KEY_VAULT_NAME: ${{ inputs.key-vault-name }} - KEY_NAME: ${{ inputs.key-name }} - run: | - set -euo pipefail - - case "$SETUP_MODE" in - signing|tools-only) - ;; - *) - echo "setup-mode must be 'signing' or 'tools-only', got '$SETUP_MODE'." >&2 - exit 1 - ;; - esac - - for variable_name in RCODESIGN_SHA256 AKV_PKCS11_LIBRARY_SHA256; do - value="${!variable_name}" - if [[ ! "$value" =~ ^[0-9a-f]{64}$ ]]; then - echo "$variable_name must be a lowercase SHA-256 digest." >&2 - exit 1 - fi - done - - for variable_name in RCODESIGN_BLOB_URI AKV_PKCS11_LIBRARY_BLOB_URI; do - value="${!variable_name}" - if [[ ! "$value" =~ ^az://[^/]+/[^/]+/.+ ]]; then - echo "$variable_name must use az:////." >&2 - exit 1 - fi - done - - if [[ "$SETUP_MODE" == "signing" ]]; then - for variable_name in \ - KEY_VAULT_NAME \ - KEY_NAME; do - if [[ -z "${!variable_name}" ]]; then - echo "$variable_name is required for AKV PKCS11 signing." >&2 - exit 1 - fi - done - fi - - - name: Resolve signing tool paths - id: paths - shell: bash - run: | - set -euo pipefail - - if [[ "${RUNNER_OS}" != "Linux" ]]; then - echo "Prebuilt AKV PKCS11 signing tools are only vendored for Linux runners, got ${RUNNER_OS}." >&2 - exit 1 - fi - - if [[ "${RUNNER_ARCH}" != "X64" && "${RUNNER_ARCH}" != "AMD64" ]]; then - echo "Prebuilt AKV PKCS11 signing tools are only vendored for amd64 runners, got ${RUNNER_ARCH}." >&2 - exit 1 - fi - - provider_root="${RUNNER_TEMP}/akv-pkcs11-provider" - rcodesign_root="${RUNNER_TEMP}/rcodesign-root" - signing_certificate_pem="${RUNNER_TEMP}/akv-signing-cert.pem" - library_name="libakv_pkcs_11.so" - - mkdir -p "$provider_root" "$rcodesign_root/bin" - - { - echo "pkcs11_library=$provider_root/$library_name" - echo "pkcs11_manifest=$provider_root/akv-pkcs11-provider.manifest" - echo "rcodesign_root=$rcodesign_root" - echo "rcodesign=$rcodesign_root/bin/rcodesign" - echo "signing_certificate_pem=$signing_certificate_pem" - } >> "$GITHUB_OUTPUT" - - - name: Validate Azure credentials for private signing artifacts - shell: bash - env: - AZURE_CLIENT_ID: ${{ inputs.azure-client-id }} - AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }} - AZURE_SUBSCRIPTION_ID: ${{ inputs.azure-subscription-id }} - run: | - set -euo pipefail - - for variable_name in AZURE_CLIENT_ID AZURE_TENANT_ID AZURE_SUBSCRIPTION_ID; do - if [[ -z "${!variable_name}" ]]; then - echo "$variable_name is required for private AKV PKCS11 signing artifacts." >&2 - exit 1 - fi - done - - - name: Log in to Azure with GitHub OIDC - uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3.0.0 - with: - client-id: ${{ inputs.azure-client-id }} - tenant-id: ${{ inputs.azure-tenant-id }} - subscription-id: ${{ inputs.azure-subscription-id }} - - - name: Install prebuilt signing tools - shell: bash - env: - RCODESIGN_BLOB_URI: ${{ inputs.rcodesign-blob-uri }} - RCODESIGN_SHA256: ${{ inputs.rcodesign-sha256 }} - RCODESIGN: ${{ steps.paths.outputs.rcodesign }} - AKV_PKCS11_LIBRARY_BLOB_URI: ${{ inputs.akv-pkcs11-library-blob-uri }} - AKV_PKCS11_LIBRARY_SHA256: ${{ inputs.akv-pkcs11-library-sha256 }} - PKCS11_LIBRARY: ${{ steps.paths.outputs.pkcs11_library }} - PKCS11_MANIFEST: ${{ steps.paths.outputs.pkcs11_manifest }} - run: | - set -euo pipefail - - download_az_blob_uri() { - local uri="$1" - local destination="$2" - local rest account container blob - - rest="${uri#az://}" - account="${rest%%/*}" - rest="${rest#*/}" - container="${rest%%/*}" - blob="${rest#*/}" - - if [[ -z "$account" || -z "$container" || -z "$blob" || "$blob" == "$rest" ]]; then - echo "Invalid Azure Blob URI. Expected az:////." >&2 - exit 1 - fi - - mkdir -p "$(dirname "$destination")" - rm -f "$destination" - if ! az storage blob download \ - --account-name "$account" \ - --container-name "$container" \ - --name "$blob" \ - --file "$destination" \ - --auth-mode login \ - --only-show-errors \ - >/dev/null 2>&1; then - echo "Failed to download a private signing artifact from Azure Blob Storage." >&2 - exit 1 - fi - } - - verify_sha256() { - local path="$1" - local expected="$2" - local actual - - actual="$(shasum -a 256 "$path" | awk '{ print $1 }')" - if [[ "$actual" != "$expected" ]]; then - echo "SHA-256 verification failed for '$path'." >&2 - exit 1 - fi - } - - echo "Downloading prebuilt rcodesign." - download_az_blob_uri "$RCODESIGN_BLOB_URI" "$RCODESIGN" - verify_sha256 "$RCODESIGN" "$RCODESIGN_SHA256" - chmod 0755 "$RCODESIGN" - - echo "Downloading prebuilt AKV PKCS11 provider." - download_az_blob_uri "$AKV_PKCS11_LIBRARY_BLOB_URI" "$PKCS11_LIBRARY" - verify_sha256 "$PKCS11_LIBRARY" "$AKV_PKCS11_LIBRARY_SHA256" - chmod 0644 "$PKCS11_LIBRARY" - - { - echo "runner_os=$RUNNER_OS" - echo "runner_arch=$RUNNER_ARCH" - echo "library_name=$(basename "$PKCS11_LIBRARY")" - } > "$PKCS11_MANIFEST" - - - name: Verify downloaded signing tools - shell: bash - env: - RCODESIGN: ${{ steps.paths.outputs.rcodesign }} - RCODESIGN_SHA256: ${{ inputs.rcodesign-sha256 }} - PKCS11_LIBRARY: ${{ steps.paths.outputs.pkcs11_library }} - AKV_PKCS11_LIBRARY_SHA256: ${{ inputs.akv-pkcs11-library-sha256 }} - PKCS11_MANIFEST: ${{ steps.paths.outputs.pkcs11_manifest }} - run: | - set -euo pipefail - - verify_sha256() { - local path="$1" - local expected="$2" - local actual - - actual="$(shasum -a 256 "$path" | awk '{ print $1 }')" - if [[ "$actual" != "$expected" ]]; then - echo "SHA-256 verification failed for '$path'." >&2 - exit 1 - fi - } - - if [[ ! -x "$RCODESIGN" ]]; then - echo "rcodesign is missing or not executable at '$RCODESIGN'." >&2 - exit 1 - fi - - if [[ ! -f "$PKCS11_LIBRARY" ]]; then - echo "AKV PKCS11 provider library is missing at '$PKCS11_LIBRARY'." >&2 - exit 1 - fi - - verify_sha256 "$RCODESIGN" "$RCODESIGN_SHA256" - verify_sha256 "$PKCS11_LIBRARY" "$AKV_PKCS11_LIBRARY_SHA256" - - "$RCODESIGN" --version - "$RCODESIGN" notarize --help > /dev/null - - if [[ -f "$PKCS11_MANIFEST" ]]; then - echo "AKV PKCS11 provider artifact manifest is present." - else - echo "AKV PKCS11 provider artifact manifest is absent." >&2 - exit 1 - fi - - - name: Download signing certificate from Key Vault - if: ${{ inputs.setup-mode == 'signing' }} - shell: bash - env: - KEY_VAULT_NAME: ${{ inputs.key-vault-name }} - KEY_NAME: ${{ inputs.key-name }} - KEY_VERSION: ${{ inputs.key-version }} - CERTIFICATE_SHA256: ${{ inputs.certificate-sha256 }} - SIGNING_CERTIFICATE_PEM: ${{ steps.paths.outputs.signing_certificate_pem }} - run: | - set -euo pipefail - - certificate_version_args=() - if [[ -n "$KEY_VERSION" ]]; then - certificate_version_args+=(--version "$KEY_VERSION") - fi - - if ! az keyvault certificate download \ - --vault-name "$KEY_VAULT_NAME" \ - --name "$KEY_NAME" \ - "${certificate_version_args[@]}" \ - --file "$SIGNING_CERTIFICATE_PEM" \ - --encoding PEM \ - --only-show-errors \ - >/dev/null 2>&1; then - echo "Failed to download the public signing certificate from Azure Key Vault." >&2 - exit 1 - fi - - if [[ -n "$CERTIFICATE_SHA256" ]]; then - actual_sha256="$( - openssl x509 -in "$SIGNING_CERTIFICATE_PEM" -noout -fingerprint -sha256 | - awk -F= '{ print toupper($2) }' | - tr -d ':\r\n' - )" - expected_sha256="$(printf '%s' "$CERTIFICATE_SHA256" | tr '[:lower:]' '[:upper:]' | tr -d ':\r\n ')" - if [[ "$actual_sha256" != "$expected_sha256" ]]; then - echo "Downloaded signing certificate SHA-256 did not match the expected fingerprint." >&2 - exit 1 - fi - fi - - - name: Export AKV PKCS11 signing environment - if: ${{ inputs.setup-mode == 'signing' }} - shell: bash - env: - RCODESIGN_ROOT: ${{ steps.paths.outputs.rcodesign_root }} - PKCS11_LIBRARY: ${{ steps.paths.outputs.pkcs11_library }} - SIGNING_CERTIFICATE_PEM: ${{ steps.paths.outputs.signing_certificate_pem }} - KEY_VAULT_NAME: ${{ inputs.key-vault-name }} - KEY_NAME: ${{ inputs.key-name }} - KEY_VERSION: ${{ inputs.key-version }} - run: | - set -euo pipefail - - { - echo "$RCODESIGN_ROOT/bin" - } >> "$GITHUB_PATH" - - { - echo "OAI_CODESIGN_BACKEND=akv-pkcs11" - echo "OAI_AKV_PKCS11_LIBRARY=$PKCS11_LIBRARY" - echo "OAI_AKV_SIGNING_CERTIFICATE_PEM=$SIGNING_CERTIFICATE_PEM" - echo "OAI_AKV_KEY_LABEL=$KEY_NAME" - echo "AZURE_CREDENTIAL_KIND=azurecli" - echo "AZURE_KEYVAULT_NAME=$KEY_VAULT_NAME" - if [[ -n "$KEY_VERSION" ]]; then - echo "AZURE_KEYVAULT_KEY_VERSION=$KEY_VERSION" - fi - } >> "$GITHUB_ENV" diff --git a/.github/actions/setup-rusty-v8/action.yml b/.github/actions/setup-rusty-v8-musl/action.yml similarity index 71% rename from .github/actions/setup-rusty-v8/action.yml rename to .github/actions/setup-rusty-v8-musl/action.yml index d9c4484657c..fbec1feb463 100644 --- a/.github/actions/setup-rusty-v8/action.yml +++ b/.github/actions/setup-rusty-v8-musl/action.yml @@ -1,20 +1,29 @@ -name: setup-rusty-v8 -description: Download and verify Codex-built rusty_v8 artifacts for Cargo builds. +name: setup-rusty-v8-musl +description: Download and verify musl rusty_v8 artifacts for Cargo builds. inputs: target: - description: Rust target triple with Codex-built V8 release artifacts. + description: Rust musl target triple. required: true runs: using: composite steps: - - name: Configure rusty_v8 artifact overrides and verify checksums + - name: Configure musl rusty_v8 artifact overrides and verify checksums shell: bash env: TARGET: ${{ inputs.target }} run: | set -euo pipefail + case "${TARGET}" in + x86_64-unknown-linux-musl|aarch64-unknown-linux-musl) + ;; + *) + echo "Unsupported musl rusty_v8 target: ${TARGET}" >&2 + exit 1 + ;; + esac + version="$(python3 "${GITHUB_WORKSPACE}/.github/scripts/rusty_v8_bazel.py" resolved-v8-crate-version)" release_tag="rusty-v8-v${version}" base_url="https://github.com/openai/codex/releases/download/${release_tag}" @@ -33,10 +42,6 @@ runs: exit 1 fi - if command -v sha256sum >/dev/null 2>&1; then - (cd "${binding_dir}" && sha256sum -c "${checksums_path}") - else - (cd "${binding_dir}" && shasum -a 256 -c "${checksums_path}") - fi + (cd "${binding_dir}" && sha256sum -c "${checksums_path}") echo "RUSTY_V8_ARCHIVE=${archive_path}" >> "${GITHUB_ENV}" echo "RUSTY_V8_SRC_BINDING_PATH=${binding_path}" >> "${GITHUB_ENV}" diff --git a/.github/dotslash-config.json b/.github/dotslash-config.json index 78ea6b37e6f..a0297c269a8 100644 --- a/.github/dotslash-config.json +++ b/.github/dotslash-config.json @@ -3,56 +3,56 @@ "codex": { "platforms": { "macos-aarch64": { - "regex": "^codex-package-aarch64-apple-darwin\\.tar\\.zst$", - "path": "bin/codex" + "regex": "^codex-aarch64-apple-darwin\\.zst$", + "path": "codex" }, "macos-x86_64": { - "regex": "^codex-package-x86_64-apple-darwin\\.tar\\.zst$", - "path": "bin/codex" + "regex": "^codex-x86_64-apple-darwin\\.zst$", + "path": "codex" }, "linux-x86_64": { - "regex": "^codex-package-x86_64-unknown-linux-musl\\.tar\\.zst$", - "path": "bin/codex" + "regex": "^codex-x86_64-unknown-linux-musl-bundle\\.tar\\.zst$", + "path": "codex" }, "linux-aarch64": { - "regex": "^codex-package-aarch64-unknown-linux-musl\\.tar\\.zst$", - "path": "bin/codex" + "regex": "^codex-aarch64-unknown-linux-musl-bundle\\.tar\\.zst$", + "path": "codex" }, "windows-x86_64": { - "regex": "^codex-package-x86_64-pc-windows-msvc\\.tar\\.zst$", - "path": "bin/codex.exe" + "regex": "^codex-x86_64-pc-windows-msvc\\.exe\\.zst$", + "path": "codex.exe" }, "windows-aarch64": { - "regex": "^codex-package-aarch64-pc-windows-msvc\\.tar\\.zst$", - "path": "bin/codex.exe" + "regex": "^codex-aarch64-pc-windows-msvc\\.exe\\.zst$", + "path": "codex.exe" } } }, "codex-app-server": { "platforms": { "macos-aarch64": { - "regex": "^codex-app-server-package-aarch64-apple-darwin\\.tar\\.zst$", - "path": "bin/codex-app-server" + "regex": "^codex-app-server-aarch64-apple-darwin\\.zst$", + "path": "codex-app-server" }, "macos-x86_64": { - "regex": "^codex-app-server-package-x86_64-apple-darwin\\.tar\\.zst$", - "path": "bin/codex-app-server" + "regex": "^codex-app-server-x86_64-apple-darwin\\.zst$", + "path": "codex-app-server" }, "linux-x86_64": { - "regex": "^codex-app-server-package-x86_64-unknown-linux-musl\\.tar\\.zst$", - "path": "bin/codex-app-server" + "regex": "^codex-app-server-x86_64-unknown-linux-musl\\.zst$", + "path": "codex-app-server" }, "linux-aarch64": { - "regex": "^codex-app-server-package-aarch64-unknown-linux-musl\\.tar\\.zst$", - "path": "bin/codex-app-server" + "regex": "^codex-app-server-aarch64-unknown-linux-musl\\.zst$", + "path": "codex-app-server" }, "windows-x86_64": { - "regex": "^codex-app-server-package-x86_64-pc-windows-msvc\\.tar\\.zst$", - "path": "bin/codex-app-server.exe" + "regex": "^codex-app-server-x86_64-pc-windows-msvc\\.exe\\.zst$", + "path": "codex-app-server.exe" }, "windows-aarch64": { - "regex": "^codex-app-server-package-aarch64-pc-windows-msvc\\.tar\\.zst$", - "path": "bin/codex-app-server.exe" + "regex": "^codex-app-server-aarch64-pc-windows-msvc\\.exe\\.zst$", + "path": "codex-app-server.exe" } } }, diff --git a/.github/dotslash-zsh-config.json b/.github/dotslash-zsh-config.json index 37285f19e24..db2c4164015 100644 --- a/.github/dotslash-zsh-config.json +++ b/.github/dotslash-zsh-config.json @@ -7,11 +7,6 @@ "format": "tar.gz", "path": "codex-zsh/bin/zsh" }, - "macos-x86_64": { - "name": "codex-zsh-x86_64-apple-darwin.tar.gz", - "format": "tar.gz", - "path": "codex-zsh/bin/zsh" - }, "linux-x86_64": { "name": "codex-zsh-x86_64-unknown-linux-musl.tar.gz", "format": "tar.gz", diff --git a/.github/scripts/build-codex-package-archive.sh b/.github/scripts/build-codex-package-archive.sh index 80da4cf20c9..90eae12ef07 100644 --- a/.github/scripts/build-codex-package-archive.sh +++ b/.github/scripts/build-codex-package-archive.sh @@ -8,9 +8,6 @@ Usage: build-codex-package-archive.sh \ --bundle \ --entrypoint-dir \ --archive-dir \ - [--bwrap-bin ] \ - [--codex-command-runner-bin ] \ - [--codex-windows-sandbox-setup-bin ] \ [--target-suffixed-entrypoint] EOF } @@ -20,10 +17,6 @@ bundle="" entrypoint_dir="" archive_dir="" target_suffixed_entrypoint="false" -resource_args=() -bwrap_bin_provided="false" -command_runner_bin_provided="false" -sandbox_setup_bin_provided="false" while [[ $# -gt 0 ]]; do case "$1" in @@ -43,27 +36,6 @@ while [[ $# -gt 0 ]]; do archive_dir="${2:?--archive-dir requires a value}" shift 2 ;; - --bwrap-bin) - resource_args+=(--bwrap-bin "${2:?--bwrap-bin requires a value}") - bwrap_bin_provided="true" - shift 2 - ;; - --codex-command-runner-bin) - resource_args+=( - --codex-command-runner-bin - "${2:?--codex-command-runner-bin requires a value}" - ) - command_runner_bin_provided="true" - shift 2 - ;; - --codex-windows-sandbox-setup-bin) - resource_args+=( - --codex-windows-sandbox-setup-bin - "${2:?--codex-windows-sandbox-setup-bin requires a value}" - ) - sandbox_setup_bin_provided="true" - shift 2 - ;; --target-suffixed-entrypoint) target_suffixed_entrypoint="true" shift @@ -114,25 +86,6 @@ if [[ "$target_suffixed_entrypoint" == "true" ]]; then entrypoint_name="${entrypoint_name}-${target}" fi -case "$target" in - *linux*) - bwrap_bin="${entrypoint_dir%/}/bwrap" - if [[ "$bwrap_bin_provided" == "false" && -f "$bwrap_bin" ]]; then - resource_args+=(--bwrap-bin "$bwrap_bin") - fi - ;; - *windows*) - command_runner_bin="${entrypoint_dir%/}/codex-command-runner.exe" - sandbox_setup_bin="${entrypoint_dir%/}/codex-windows-sandbox-setup.exe" - if [[ "$command_runner_bin_provided" == "false" && -f "$command_runner_bin" ]]; then - resource_args+=(--codex-command-runner-bin "$command_runner_bin") - fi - if [[ "$sandbox_setup_bin_provided" == "false" && -f "$sandbox_setup_bin" ]]; then - resource_args+=(--codex-windows-sandbox-setup-bin "$sandbox_setup_bin") - fi - ;; -esac - repo_root="${GITHUB_WORKSPACE:-}" if [[ -z "$repo_root" ]]; then repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" @@ -144,29 +97,16 @@ else python_bin="python" fi -if ! command -v zstd >/dev/null 2>&1 && [[ -x "${repo_root}/.github/workflows/zstd" ]]; then - export PATH="${repo_root}/.github/workflows:${PATH}" -fi - mkdir -p "$archive_dir" package_dir="${RUNNER_TEMP:-/tmp}/${archive_stem}-${target}" -gzip_archive_path="${archive_dir}/${archive_stem}-${target}.tar.gz" -zstd_archive_path="${archive_dir}/${archive_stem}-${target}.tar.zst" +archive_path="${archive_dir}/${archive_stem}-${target}.tar.gz" rm -rf "$package_dir" -python_args=( - "${repo_root}/scripts/build_codex_package.py" - --target "$target" - --variant "$variant" - --entrypoint-bin "${entrypoint_dir%/}/${entrypoint_name}${exe_suffix}" - --cargo-profile release - --package-dir "$package_dir" - --archive-output "$gzip_archive_path" - --archive-output "$zstd_archive_path" -) -if ((${#resource_args[@]} > 0)); then - python_args+=("${resource_args[@]}") -fi -python_args+=(--force) - -"$python_bin" "${python_args[@]}" +"$python_bin" "${repo_root}/scripts/build_codex_package.py" \ + --target "$target" \ + --variant "$variant" \ + --entrypoint-bin "${entrypoint_dir%/}/${entrypoint_name}${exe_suffix}" \ + --cargo-profile release \ + --package-dir "$package_dir" \ + --archive-output "$archive_path" \ + --force diff --git a/.github/scripts/install-musl-build-tools.sh b/.github/scripts/install-musl-build-tools.sh index 49035f53911..e4c6683d0e6 100644 --- a/.github/scripts/install-musl-build-tools.sh +++ b/.github/scripts/install-musl-build-tools.sh @@ -150,9 +150,7 @@ for arg in "\$@"; do args+=("\${arg}") done -# Zig enables UBSan for debug C builds by default. Rust links these objects -# without Zig's sanitizer runtime, so keep native dependencies uninstrumented. -exec "${zig_bin}" cc -target "${zig_target}" "\${args[@]}" -fno-sanitize=undefined +exec "${zig_bin}" cc -target "${zig_target}" "\${args[@]}" EOF cat >"${cxx}" <> "$GITHUB_ENV" pkg_config_path_var="PKG_CONFIG_PATH_${TARGET}" pkg_config_path_var="${pkg_config_path_var//-/_}" echo "${pkg_config_path_var}=${libcap_pkgconfig_dir}" >> "$GITHUB_ENV" -pkg_config_libdir_var="PKG_CONFIG_LIBDIR_${TARGET}" -pkg_config_libdir_var="${pkg_config_libdir_var//-/_}" -# Do not let musl cross-builds resolve native libraries from the host glibc -# pkg-config directories. libcap is the only target package provided here. -echo "${pkg_config_libdir_var}=${libcap_pkgconfig_dir}" >> "$GITHUB_ENV" if [[ -n "${sysroot}" && "${sysroot}" != "/" ]]; then echo "PKG_CONFIG_SYSROOT_DIR=${sysroot}" >> "$GITHUB_ENV" diff --git a/.github/scripts/macos-signing/notarize_macos_binary_with_rcodesign.sh b/.github/scripts/macos-signing/notarize_macos_binary_with_rcodesign.sh deleted file mode 100755 index 8ebe490d41e..00000000000 --- a/.github/scripts/macos-signing/notarize_macos_binary_with_rcodesign.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env bash - -# Submits a signed standalone macOS binary to Apple notarization through -# rcodesign. Standalone binaries cannot carry a stapled ticket, so the binary -# is submitted in a ZIP and the successful notarization log is retained. - -set -euo pipefail - -usage() { - cat >&2 <<'EOF' -Usage: notarize_macos_binary_with_rcodesign.sh --binary PATH [--report-dir PATH] [--max-wait-seconds SECONDS] - -Options: - --binary PATH Signed standalone macOS binary to notarize. - --report-dir PATH Directory for notarization logs. - --max-wait-seconds SECONDS Maximum rcodesign notarization wait time. -EOF -} - -binary_path="" -report_dir="${RUNNER_TEMP:-/tmp}/macos-binary-notarization-verification" -max_wait_seconds="600" - -while [[ $# -gt 0 ]]; do - case "$1" in - --binary) - binary_path="${2:-}" - shift 2 - ;; - --report-dir) - report_dir="${2:-}" - shift 2 - ;; - --max-wait-seconds) - max_wait_seconds="${2:-}" - shift 2 - ;; - -h|--help) - usage - exit 0 - ;; - *) - echo "Unknown notarization argument: $1" >&2 - usage - exit 2 - ;; - esac -done - -if [[ -z "$binary_path" ]]; then - echo "--binary is required." >&2 - usage - exit 2 -fi - -if [[ ! -f "$binary_path" ]]; then - echo "Binary does not exist: $binary_path" >&2 - exit 1 -fi - -if [[ ! "$max_wait_seconds" =~ ^[0-9]+$ ]]; then - echo "--max-wait-seconds must be a non-negative integer." >&2 - exit 2 -fi - -for command_name in rcodesign zip; do - if ! command -v "$command_name" >/dev/null 2>&1; then - echo "$command_name was not found on PATH." >&2 - exit 1 - fi -done - -missing_environment=0 -for variable_name in \ - APPLE_NOTARIZATION_ISSUER_ID \ - APPLE_NOTARIZATION_KEY_ID \ - APPLE_NOTARIZATION_KEY_P8 -do - if [[ -z "${!variable_name:-}" ]]; then - echo "$variable_name must be set from CI secrets before notarizing a binary." >&2 - missing_environment=1 - fi -done - -if [[ "$missing_environment" -ne 0 ]]; then - exit 2 -fi - -mkdir -p "$report_dir" - -notarization_temp_dir="$(mktemp -d)" -trap 'rm -rf "$notarization_temp_dir" >/dev/null' EXIT - -private_key_path="$notarization_temp_dir/AuthKey_${APPLE_NOTARIZATION_KEY_ID}.p8" -if ! printf '%s' "$APPLE_NOTARIZATION_KEY_P8" | base64 --decode >"$private_key_path" 2>/dev/null; then - if ! printf '%s' "$APPLE_NOTARIZATION_KEY_P8" | base64 -D >"$private_key_path" 2>/dev/null; then - echo "APPLE_NOTARIZATION_KEY_P8 must be a base64-encoded .p8 private key." >&2 - exit 2 - fi -fi -chmod 600 "$private_key_path" - -api_key_path="$notarization_temp_dir/app-store-connect-api-key.json" -rcodesign encode-app-store-connect-api-key \ - --output-path "$api_key_path" \ - "$APPLE_NOTARIZATION_ISSUER_ID" \ - "$APPLE_NOTARIZATION_KEY_ID" \ - "$private_key_path" \ - >"$report_dir/encode-app-store-connect-api-key.log" 2>&1 - -binary_name="$(basename "$binary_path")" -archive_path="$notarization_temp_dir/${binary_name}.zip" -( - cd "$(dirname "$binary_path")" - zip -q "$archive_path" "$binary_name" -) - -notarization_log="$report_dir/${binary_name}-notarization.log" -rcodesign notarize \ - --api-key-file "$api_key_path" \ - --max-wait-seconds "$max_wait_seconds" \ - --wait \ - "$archive_path" \ - 2>&1 | tee "$notarization_log" - -{ - echo "binary_name=$binary_name" - echo "max_wait_seconds=$max_wait_seconds" - echo "binary_sha256=$(shasum -a 256 "$binary_path" | awk '{ print $1 }')" - echo "rcodesign_notarize=completed" -} >"$report_dir/${binary_name}-notarization-summary.txt" diff --git a/.github/scripts/macos-signing/notarize_macos_dmg_with_rcodesign.sh b/.github/scripts/macos-signing/notarize_macos_dmg_with_rcodesign.sh deleted file mode 100755 index a1125d436aa..00000000000 --- a/.github/scripts/macos-signing/notarize_macos_dmg_with_rcodesign.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env bash - -# Notarizes and staples a signed macOS DMG through rcodesign. -# -# This is the Linux-compatible notarization path for the AKV/PKCS#11 signing -# flow. It records notarization inputs and logs so workflow artifacts can be -# audited without exposing the App Store Connect private key. - -set -euo pipefail - -usage() { - cat >&2 <<'EOF' -Usage: notarize_macos_dmg_with_rcodesign.sh --dmg PATH [--report-dir PATH] [--max-wait-seconds SECONDS] - -Options: - --dmg PATH Signed DMG to submit to Apple notarization. - --report-dir PATH Directory for notarization logs. - --max-wait-seconds SECONDS Maximum rcodesign notarization wait time. -EOF -} - -dmg_path="" -report_dir="${RUNNER_TEMP:-/tmp}/macos-notarization-verification" -max_wait_seconds="600" - -while [[ $# -gt 0 ]]; do - case "$1" in - --dmg) - dmg_path="${2:-}" - shift 2 - ;; - --report-dir) - report_dir="${2:-}" - shift 2 - ;; - --max-wait-seconds) - max_wait_seconds="${2:-}" - shift 2 - ;; - -h|--help) - usage - exit 0 - ;; - *) - echo "Unknown notarization argument: $1" >&2 - usage - exit 2 - ;; - esac -done - -if [[ -z "$dmg_path" ]]; then - echo "--dmg is required." >&2 - usage - exit 2 -fi - -if [[ ! -f "$dmg_path" ]]; then - echo "DMG does not exist: $dmg_path" >&2 - exit 1 -fi - -if [[ ! "$max_wait_seconds" =~ ^[0-9]+$ ]]; then - echo "--max-wait-seconds must be a non-negative integer." >&2 - exit 2 -fi - -if ! command -v rcodesign > /dev/null 2>&1; then - echo "rcodesign was not found on PATH." >&2 - exit 1 -fi - -missing_environment=0 -for variable_name in \ - APPLE_NOTARIZATION_ISSUER_ID \ - APPLE_NOTARIZATION_KEY_ID \ - APPLE_NOTARIZATION_KEY_P8 -do - if [[ -z "${!variable_name:-}" ]]; then - echo "$variable_name must be set from CI secrets before notarizing a DMG." >&2 - missing_environment=1 - fi -done - -if [[ "$missing_environment" -ne 0 ]]; then - exit 2 -fi - -mkdir -p "$report_dir" - -notarization_temp_dir="$(mktemp -d)" -trap 'rm -rf "$notarization_temp_dir" > /dev/null' EXIT - -private_key_path="$notarization_temp_dir/AuthKey_${APPLE_NOTARIZATION_KEY_ID}.p8" -if ! printf '%s' "$APPLE_NOTARIZATION_KEY_P8" | base64 --decode > "$private_key_path" 2> /dev/null; then - if ! printf '%s' "$APPLE_NOTARIZATION_KEY_P8" | base64 -D > "$private_key_path" 2> /dev/null; then - echo "APPLE_NOTARIZATION_KEY_P8 must be a base64-encoded .p8 private key." >&2 - exit 2 - fi -fi -chmod 600 "$private_key_path" - -api_key_path="$notarization_temp_dir/app-store-connect-api-key.json" -rcodesign encode-app-store-connect-api-key \ - --output-path "$api_key_path" \ - "$APPLE_NOTARIZATION_ISSUER_ID" \ - "$APPLE_NOTARIZATION_KEY_ID" \ - "$private_key_path" \ - > "$report_dir/encode-app-store-connect-api-key.log" 2>&1 - -notarization_log="$report_dir/dmg-notarization.log" -rcodesign notarize \ - --api-key-file "$api_key_path" \ - --max-wait-seconds "$max_wait_seconds" \ - --staple \ - "$dmg_path" \ - 2>&1 | tee "$notarization_log" - -{ - echo "dmg_path=$dmg_path" - echo "max_wait_seconds=$max_wait_seconds" - echo "dmg_sha256=$(shasum -a 256 "$dmg_path" | awk '{ print $1 }')" - echo "rcodesign_notarize_staple=completed" -} > "$report_dir/dmg-notarization-summary.txt" diff --git a/.github/scripts/macos-signing/sign_macos_code.sh b/.github/scripts/macos-signing/sign_macos_code.sh deleted file mode 100755 index 9f86741410d..00000000000 --- a/.github/scripts/macos-signing/sign_macos_code.sh +++ /dev/null @@ -1,245 +0,0 @@ -#!/usr/bin/env bash - -# Small compatibility wrapper around native codesign and rcodesign. -# -# Existing packaging scripts call this instead of choosing a signing backend -# directly. OAI_CODESIGN_BACKEND=akv-pkcs11 routes signing through rcodesign -# while preserving the option, entitlement, identifier, timestamp, and deep -# signing surface used by the native codesign path. - -set -euo pipefail - -usage() { - cat >&2 <<'EOF' -Usage: sign_macos_code.sh --target PATH --identity IDENTITY [options] - -Options: - --deep true|false - --entitlements PATH - --identifier IDENTIFIER - --identity IDENTITY - --options FLAGS - --target PATH - --timestamp true|false|none -EOF -} - -target="" -identity="" -options="" -entitlements_file="" -identifier="" -deep="false" -timestamp="true" - -while [[ $# -gt 0 ]]; do - case "$1" in - --deep) - deep="${2:-}" - shift 2 - ;; - --entitlements) - entitlements_file="${2:-}" - shift 2 - ;; - --identifier) - identifier="${2:-}" - shift 2 - ;; - --identity) - identity="${2:-}" - shift 2 - ;; - --options) - options="${2:-}" - shift 2 - ;; - --target) - target="${2:-}" - shift 2 - ;; - --timestamp) - timestamp="${2:-}" - shift 2 - ;; - -h|--help) - usage - exit 0 - ;; - *) - echo "Unknown signing argument: $1" >&2 - usage - exit 2 - ;; - esac -done - -if [[ -z "$target" ]]; then - echo "--target is required." >&2 - usage - exit 2 -fi - -if [[ ! -e "$target" ]]; then - echo "Signing target does not exist: $target" >&2 - exit 1 -fi - -case "$deep" in - true|false) ;; - *) - echo "--deep must be true or false, got '$deep'." >&2 - exit 2 - ;; -esac - -case "$timestamp" in - true|false|none) ;; - *) - echo "--timestamp must be true, false, or none, got '$timestamp'." >&2 - exit 2 - ;; -esac - -sign_with_codesign() { - if [[ -z "$identity" ]]; then - echo "Native codesign requires --identity." >&2 - exit 2 - fi - - local -a args - args=(--force) - - if [[ "$deep" == "true" ]]; then - args+=(--deep) - fi - - if [[ -n "$options" ]]; then - args+=(--options "$options") - fi - - case "$timestamp" in - true) - args+=(--timestamp) - ;; - false|none) - args+=(--timestamp=none) - ;; - esac - - if [[ -n "$entitlements_file" ]]; then - args+=(--entitlements "$entitlements_file") - fi - - if [[ -n "$identifier" ]]; then - args+=(--identifier "$identifier") - fi - - args+=(--sign "$identity" "$target") - codesign "${args[@]}" -} - -append_rcodesign_flags() { - local raw_options="$1" - local option="" - - if [[ -z "$raw_options" ]]; then - return 0 - fi - - IFS=',' read -ra split_options <<< "$raw_options" - for option in "${split_options[@]}"; do - option="${option//[[:space:]]/}" - [[ -z "$option" ]] && continue - - case "$option" in - host|hard|kill|expires|restrict|library|runtime|linker-signed) - rcodesign_args+=(--code-signature-flags "$option") - ;; - *) - echo "Unsupported rcodesign code signature option: $option" >&2 - exit 2 - ;; - esac - done -} - -rcodesign_options_require_notarization() { - local raw_options="$1" - local option="" - - if [[ -z "$raw_options" || "$timestamp" != "true" ]]; then - return 1 - fi - - IFS=',' read -ra split_options <<< "$raw_options" - for option in "${split_options[@]}"; do - option="${option//[[:space:]]/}" - if [[ "$option" == "runtime" ]]; then - return 0 - fi - done - - return 1 -} - -sign_with_rcodesign() { - : "${OAI_AKV_PKCS11_LIBRARY:?OAI_AKV_PKCS11_LIBRARY is required for AKV PKCS11 signing.}" - : "${OAI_AKV_SIGNING_CERTIFICATE_PEM:?OAI_AKV_SIGNING_CERTIFICATE_PEM is required for AKV PKCS11 signing.}" - : "${OAI_AKV_KEY_LABEL:?OAI_AKV_KEY_LABEL is required for AKV PKCS11 signing.}" - - if ! command -v rcodesign >/dev/null 2>&1; then - echo "rcodesign was not found on PATH." >&2 - exit 1 - fi - - local -a rcodesign_args - rcodesign_args=( - sign - --config-file /dev/null - --pkcs11-library "$OAI_AKV_PKCS11_LIBRARY" - --pkcs11-certificate-file "$OAI_AKV_SIGNING_CERTIFICATE_PEM" - --pkcs11-key-label "$OAI_AKV_KEY_LABEL" - ) - - if [[ "$deep" == "false" ]]; then - rcodesign_args+=(--shallow) - fi - - case "$timestamp" in - true) - ;; - false|none) - rcodesign_args+=(--timestamp-url none) - ;; - esac - - append_rcodesign_flags "$options" - if rcodesign_options_require_notarization "$options"; then - rcodesign_args+=(--for-notarization) - fi - - if [[ -n "$entitlements_file" ]]; then - rcodesign_args+=(--entitlements-xml-file "$entitlements_file") - fi - - if [[ -n "$identifier" ]]; then - rcodesign_args+=(--binary-identifier "$identifier") - fi - - rcodesign_args+=("$target") - rcodesign "${rcodesign_args[@]}" -} - -case "${OAI_CODESIGN_BACKEND:-codesign}" in - codesign|"") - sign_with_codesign - ;; - akv-pkcs11) - sign_with_rcodesign - ;; - *) - echo "Unsupported OAI_CODESIGN_BACKEND: ${OAI_CODESIGN_BACKEND}" >&2 - exit 2 - ;; -esac diff --git a/.github/scripts/run-bazel-ci.sh b/.github/scripts/run-bazel-ci.sh index 89f937a998b..f98e4d8cb99 100755 --- a/.github/scripts/run-bazel-ci.sh +++ b/.github/scripts/run-bazel-ci.sh @@ -53,20 +53,11 @@ fi run_bazel() { if [[ "${RUNNER_OS:-}" == "Windows" ]]; then - MSYS2_ARG_CONV_EXCL='*' "$(dirname "${BASH_SOURCE[0]}")/run_bazel_with_buildbuddy.py" "$@" + MSYS2_ARG_CONV_EXCL='*' bazel "$@" return fi - "$(dirname "${BASH_SOURCE[0]}")/run_bazel_with_buildbuddy.py" "$@" -} - -run_bazel_with_startup_args() { - if (( ${#bazel_startup_args[@]} > 0 )); then - run_bazel "${bazel_startup_args[@]}" "$@" - return - fi - - run_bazel "$@" + bazel "$@" } ci_config=ci-linux @@ -86,16 +77,23 @@ esac print_bazel_test_log_tails() { local console_log="$1" local testlogs_dir - + local -a bazel_info_cmd=(bazel) local -a bazel_info_args=(info) - if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then - # `bazel info` needs the same CI config as the failed test invocation so - # platform-specific output roots match. On Windows, omitting `ci-windows` - # would point at `local_windows-fastbuild` even when the test ran with the - # MSVC host platform under `local_windows_msvc-fastbuild`. - bazel_info_args+=("--config=${ci_config}") + + if (( ${#bazel_startup_args[@]} > 0 )); then + bazel_info_cmd+=("${bazel_startup_args[@]}") fi + # `bazel info` needs the same CI config as the failed test invocation so + # platform-specific output roots match. On Windows, omitting `ci-windows` + # would point at `local_windows-fastbuild` even when the test ran with the + # MSVC host platform under `local_windows_msvc-fastbuild`. + if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then + bazel_info_args+=( + "--config=${ci_config}" + "--remote_header=x-buildbuddy-api-key=${BUILDBUDDY_API_KEY}" + ) + fi # Only pass flags that affect Bazel's output-root selection or repository # lookup. Test/build-only flags such as execution logs or remote download # mode can make `bazel info` fail, which would hide the real test log path. @@ -107,7 +105,7 @@ print_bazel_test_log_tails() { esac done - testlogs_dir="$(run_bazel_with_startup_args \ + testlogs_dir="$(run_bazel "${bazel_info_cmd[@]:1}" \ --noexperimental_remote_repo_contents_cache \ "${bazel_info_args[@]}" \ bazel-testlogs 2>/dev/null || echo bazel-testlogs)" @@ -256,9 +254,8 @@ if [[ ${#bazel_args[@]} -eq 0 || ${#bazel_targets[@]} -eq 0 ]]; then fi if [[ "${RUNNER_OS:-}" == "Windows" && $windows_cross_compile -eq 1 && -z "${BUILDBUDDY_API_KEY:-}" ]]; then - # Windows cross-compilation depends on authenticated RBE. Preserve the local - # Windows build shape when credentials are unavailable. - ci_config=ci-windows + # Fork PRs do not receive the BuildBuddy secret needed for the remote + # cross-compile config. Preserve the previous local Windows build shape. windows_msvc_host_platform=1 fi @@ -300,9 +297,9 @@ if [[ "${RUNNER_OS:-}" == "Windows" && $windows_cross_compile -eq 1 && -n "${BUI fi if [[ "${RUNNER_OS:-}" == "Windows" && $windows_cross_compile -eq 1 && -z "${BUILDBUDDY_API_KEY:-}" ]]; then - # The Windows cross-compile config depends on authenticated remote - # execution. When credentials are unavailable, keep the local build shape - # and its lower concurrency cap. + # The Windows cross-compile config depends on remote execution. Fork PRs do + # not receive the BuildBuddy secret, so fall back to the existing local build + # shape and keep its lower concurrency cap. post_config_bazel_args+=(--jobs=8) fi @@ -380,31 +377,70 @@ fi bazel_console_log="$(mktemp)" trap 'rm -f "$bazel_console_log"' EXIT -bazel_run_args=( - "${bazel_args[@]}" -) +bazel_cmd=(bazel) +if (( ${#bazel_startup_args[@]} > 0 )); then + bazel_cmd+=("${bazel_startup_args[@]}") +fi + if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then echo "BuildBuddy API key is available; using remote Bazel configuration." - bazel_run_args+=("--config=${ci_config}") + # Work around Bazel 9 remote repo contents cache / overlay materialization failures + # seen in CI (for example "is not a symlink" or permission errors while + # materializing external repos such as rules_perl). We still use BuildBuddy for + # remote execution/cache; this only disables the startup-level repo contents cache. + bazel_run_args=( + "${bazel_args[@]}" + "--config=${ci_config}" + "--remote_header=x-buildbuddy-api-key=${BUILDBUDDY_API_KEY}" + ) + if (( ${#post_config_bazel_args[@]} > 0 )); then + bazel_run_args+=("${post_config_bazel_args[@]}") + fi + set +e + run_bazel "${bazel_cmd[@]:1}" \ + --noexperimental_remote_repo_contents_cache \ + "${bazel_run_args[@]}" \ + -- \ + "${bazel_targets[@]}" \ + 2>&1 | tee "$bazel_console_log" + bazel_status=${PIPESTATUS[0]} + set -e else echo "BuildBuddy API key is not available; using local Bazel configuration." + # Keep fork/community PRs on Bazel but disable remote services that are + # configured in .bazelrc and require auth. + # + # Flag docs: + # - Command-line reference: https://bazel.build/reference/command-line-reference + # - Remote caching overview: https://bazel.build/remote/caching + # - Remote execution overview: https://bazel.build/remote/rbe + # - Build Event Protocol overview: https://bazel.build/remote/bep + # + # --noexperimental_remote_repo_contents_cache: + # disable remote repo contents cache enabled in .bazelrc startup options. + # https://bazel.build/reference/command-line-reference#startup_options-flag--experimental_remote_repo_contents_cache + # --remote_cache= and --remote_executor=: + # clear remote cache/execution endpoints configured in .bazelrc. + # https://bazel.build/reference/command-line-reference#common_options-flag--remote_cache + # https://bazel.build/reference/command-line-reference#common_options-flag--remote_executor + bazel_run_args=( + "${bazel_args[@]}" + --remote_cache= + --remote_executor= + ) + if (( ${#post_config_bazel_args[@]} > 0 )); then + bazel_run_args+=("${post_config_bazel_args[@]}") + fi + set +e + run_bazel "${bazel_cmd[@]:1}" \ + --noexperimental_remote_repo_contents_cache \ + "${bazel_run_args[@]}" \ + -- \ + "${bazel_targets[@]}" \ + 2>&1 | tee "$bazel_console_log" + bazel_status=${PIPESTATUS[0]} + set -e fi -if (( ${#post_config_bazel_args[@]} > 0 )); then - bazel_run_args+=("${post_config_bazel_args[@]}") -fi -set +e -# Work around Bazel 9 remote repo contents cache / overlay materialization -# failures seen in CI (for example "is not a symlink" or permission errors -# while materializing external repos such as rules_perl). This only disables -# the startup-level repo contents cache; keyed runs still use BuildBuddy. -run_bazel_with_startup_args \ - --noexperimental_remote_repo_contents_cache \ - "${bazel_run_args[@]}" \ - -- \ - "${bazel_targets[@]}" \ - 2>&1 | tee "$bazel_console_log" -bazel_status=${PIPESTATUS[0]} -set -e if [[ ${bazel_status:-0} -ne 0 ]]; then if [[ $print_failed_bazel_action_summary -eq 1 ]]; then diff --git a/.github/scripts/run-bazel-query-ci.sh b/.github/scripts/run-bazel-query-ci.sh index f5d4f56f49c..dd03b671692 100755 --- a/.github/scripts/run-bazel-query-ci.sh +++ b/.github/scripts/run-bazel-query-ci.sh @@ -2,17 +2,48 @@ set -euo pipefail -# Run target-discovery queries with the same startup settings as the main -# build/test invocation so they can reuse the same Bazel server. Queries only -# enumerate labels, so they intentionally do not select CI or remote configs. +# Run Bazel queries with the same CI startup settings as the main build/test +# invocation so target-discovery queries can reuse the same Bazel server. -if [[ $# -lt 2 || "${@: -2:1}" != "--" ]]; then - echo "Usage: $0 [...] -- " >&2 +query_args=() +windows_cross_compile=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --windows-cross-compile) + windows_cross_compile=1 + shift + ;; + --) + shift + break + ;; + *) + query_args+=("$1") + shift + ;; + esac +done + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 [--windows-cross-compile] [...] -- " >&2 exit 1 fi -query_args=("${@:1:$#-2}") -query_expression="${@: -1}" +query_expression="$1" + +ci_config=ci-linux +case "${RUNNER_OS:-}" in + macOS) + ci_config=ci-macos + ;; + Windows) + if [[ $windows_cross_compile -eq 1 ]]; then + ci_config=ci-windows-cross + else + ci_config=ci-windows + fi + ;; +esac bazel_startup_args=() if [[ -n "${BAZEL_OUTPUT_USER_ROOT:-}" ]]; then @@ -29,6 +60,12 @@ run_bazel() { } bazel_query_args=(--noexperimental_remote_repo_contents_cache query) +if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then + bazel_query_args+=( + "--config=${ci_config}" + "--remote_header=x-buildbuddy-api-key=${BUILDBUDDY_API_KEY}" + ) +fi if [[ -n "${BAZEL_REPO_CONTENTS_CACHE:-}" ]]; then bazel_query_args+=("--repo_contents_cache=${BAZEL_REPO_CONTENTS_CACHE}") @@ -38,10 +75,7 @@ if [[ -n "${BAZEL_REPOSITORY_CACHE:-}" ]]; then bazel_query_args+=("--repository_cache=${BAZEL_REPOSITORY_CACHE}") fi -if (( ${#query_args[@]} > 0 )); then - bazel_query_args+=("${query_args[@]}") -fi -bazel_query_args+=("$query_expression") +bazel_query_args+=("${query_args[@]}" "$query_expression") if (( ${#bazel_startup_args[@]} > 0 )); then run_bazel "${bazel_startup_args[@]}" "${bazel_query_args[@]}" diff --git a/.github/scripts/run_bazel_with_buildbuddy.py b/.github/scripts/run_bazel_with_buildbuddy.py deleted file mode 100755 index f9f329d6869..00000000000 --- a/.github/scripts/run_bazel_with_buildbuddy.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python3 - -import json -import os -import subprocess -import sys -from collections.abc import Mapping -from collections.abc import Sequence -from pathlib import Path - - -OPENAI_REPOSITORY = "openai/codex" -# Remote configurations select cache/BES/download endpoints. Their -rbe forms -# also select the matching remote executor endpoint. -GENERIC_REMOTE_CONFIG = "buildbuddy-generic" -OPENAI_REMOTE_CONFIG = "buildbuddy-openai" -# These CI configurations require remote build execution. The wrapper supplies -# an RBE configuration, which also includes the common `remote` settings. -REMOTE_EXECUTION_CONFIGS = { - "--config=ci-linux", - "--config=ci-macos", - "--config=ci-v8", - "--config=ci-windows-cross", -} -# Only authenticated workflow runs executing trusted upstream code may use the -# OpenAI BuildBuddy host. A pull request event without proof that its head is -# in the upstream repository fails closed to the generic host. -def is_trusted_upstream_run(env: Mapping[str, str]) -> bool: - # `GITHUB_REPOSITORY` is easy to set locally. Requiring GitHub's workflow - # marker prevents a local command from opting itself into the OpenAI host. - if ( - env.get("GITHUB_ACTIONS") != "true" - or env.get("GITHUB_REPOSITORY") != OPENAI_REPOSITORY - ): - return False - # Non-PR workflow runs in `openai/codex` execute upstream refs, so they are - # trusted. Fork code reaches these workflows only through pull requests. - if env.get("GITHUB_EVENT_NAME") != "pull_request": - return True - - event_path = env.get("GITHUB_EVENT_PATH") - if not event_path: - return False - try: - event = json.loads(Path(event_path).read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return False - - try: - return event["pull_request"]["head"]["repo"]["fork"] is False - except (KeyError, TypeError): - return False - - -def uses_openai_host(env: Mapping[str, str]) -> bool: - return bool(env.get("BUILDBUDDY_API_KEY")) and is_trusted_upstream_run(env) - - -def uses_remote_execution(args: Sequence[str]) -> bool: - try: - separator_idx = args.index("--") - except ValueError: - separator_idx = len(args) - return any(arg in REMOTE_EXECUTION_CONFIGS for arg in args[:separator_idx]) - - -def remote_config(args: Sequence[str], env: Mapping[str, str]) -> str | None: - if not env.get("BUILDBUDDY_API_KEY"): - return None - - config = OPENAI_REMOTE_CONFIG if uses_openai_host(env) else GENERIC_REMOTE_CONFIG - if uses_remote_execution(args): - config += "-rbe" - return config - - -def bazel_args_without_remote_execution(args: Sequence[str]) -> list[str]: - # Remote CI configs require BuildBuddy credentials. Removing them preserves - # the local fallback used for fork pull requests. - try: - separator_idx = args.index("--") - except ValueError: - separator_idx = len(args) - return [ - *(arg for arg in args[:separator_idx] if arg not in REMOTE_EXECUTION_CONFIGS), - *args[separator_idx:], - ] - - -def bazel_args_with_remote_config( - args: Sequence[str], env: Mapping[str, str] -) -> list[str]: - config = remote_config(args, env) - if config is None: - return bazel_args_without_remote_execution(args) - - # `remote_config()` returns a configuration only when this key is present. - api_key = env["BUILDBUDDY_API_KEY"] - remote_args = [ - f"--config={config}", - f"--remote_header=x-buildbuddy-api-key={api_key}", - ] - - # Insert immediately after the Bazel command. This keeps wrapper-added - # options out of positional payloads and lets later CI configs override - # shared RBE defaults such as the Windows cross-compilation exec platforms. - insertion_idx = next( - (idx + 1 for idx, arg in enumerate(args) if not arg.startswith("-")), - len(args), - ) - return [*args[:insertion_idx], *remote_args, *args[insertion_idx:]] - - -def bazel_command(*args: str, env: Mapping[str, str] | None = None) -> list[str]: - env = os.environ if env is None else env - bazel = env.get("CODEX_BAZEL_BIN", "bazel") - return [bazel, *bazel_args_with_remote_config(args, env)] - - -def main() -> None: - config = remote_config(sys.argv[1:], os.environ) - if config is None: - print( - "BuildBuddy key unavailable; using local Bazel configuration.", - file=sys.stderr, - ) - else: - host_description = ( - "OpenAI tenant" if uses_openai_host(os.environ) else "generic" - ) - print( - f"Using {host_description} BuildBuddy configuration: {config}.", - file=sys.stderr, - ) - - command = bazel_command(*sys.argv[1:]) - if os.name == "nt": - # Windows CRT exec can split arguments containing spaces and lose the - # eventual child exit status. Wait for Bazel and propagate its status. - result = subprocess.run(command, check=False) - raise SystemExit(result.returncode) - - os.execvp(command[0], command) - - -if __name__ == "__main__": - main() diff --git a/.github/scripts/rusty_v8_bazel.py b/.github/scripts/rusty_v8_bazel.py index 329d3f6c54a..2f46daf45eb 100644 --- a/.github/scripts/rusty_v8_bazel.py +++ b/.github/scripts/rusty_v8_bazel.py @@ -5,6 +5,7 @@ import argparse import gzip import hashlib +import os import re import shutil import subprocess @@ -12,7 +13,6 @@ import tomllib from pathlib import Path -from run_bazel_with_buildbuddy import bazel_command from rusty_v8_module_bazel import ( RustyV8ChecksumError, check_module_bazel, @@ -29,22 +29,33 @@ ARTIFACT_BAZEL_CONFIGS = ["rusty-v8-upstream-libcxx"] +def bazel_remote_args() -> list[str]: + buildbuddy_api_key = os.environ.get("BUILDBUDDY_API_KEY") + if not buildbuddy_api_key: + return [] + return [f"--remote_header=x-buildbuddy-api-key={buildbuddy_api_key}"] + + def bazel_execroot() -> Path: - output = subprocess.check_output( - bazel_command("info", "execution_root"), + result = subprocess.run( + ["bazel", "info", "execution_root"], cwd=ROOT, + check=True, + capture_output=True, text=True, ) - return Path(output.strip()) + return Path(result.stdout.strip()) def bazel_output_base() -> Path: - output = subprocess.check_output( - bazel_command("info", "output_base"), + result = subprocess.run( + ["bazel", "info", "output_base"], cwd=ROOT, + check=True, + capture_output=True, text=True, ) - return Path(output.strip()) + return Path(result.stdout.strip()) def bazel_output_path(path: str) -> Path: @@ -61,22 +72,24 @@ def bazel_output_files( ) -> list[Path]: expression = "set(" + " ".join(labels) + ")" bazel_configs = bazel_configs or [] - output = subprocess.check_output( - bazel_command( + result = subprocess.run( + [ + "bazel", "cquery", "-c", compilation_mode, f"--platforms=@llvm//platforms:{platform}", *[f"--config={config}" for config in bazel_configs], + *bazel_remote_args(), "--output=files", expression, - ), + ], cwd=ROOT, + check=True, + capture_output=True, text=True, ) - return [ - bazel_output_path(line.strip()) for line in output.splitlines() if line.strip() - ] + return [bazel_output_path(line.strip()) for line in result.stdout.splitlines() if line.strip()] def bazel_build( @@ -89,15 +102,17 @@ def bazel_build( bazel_configs = bazel_configs or [] download_args = ["--remote_download_toplevel"] if download_toplevel else [] subprocess.run( - bazel_command( + [ + "bazel", "build", "-c", compilation_mode, f"--platforms=@llvm//platforms:{platform}", *[f"--config={config}" for config in bazel_configs], + *bazel_remote_args(), *download_args, *labels, - ), + ], cwd=ROOT, check=True, ) @@ -157,7 +172,7 @@ def resolved_v8_crate_version() -> str: matches = sorted( set( re.findall( - r"https://static\.crates\.io/crates/v8/v8-([0-9]+\.[0-9]+\.[0-9]+)\.crate", + r'https://static\.crates\.io/crates/v8/v8-([0-9]+\.[0-9]+\.[0-9]+)\.crate', module_bazel, ) ) @@ -219,17 +234,13 @@ def stage_artifacts( output_dir: Path, sandbox: bool, ) -> None: - missing_paths = [ - str(path) for path in [lib_path, binding_path] if not path.exists() - ] + missing_paths = [str(path) for path in [lib_path, binding_path] if not path.exists()] if missing_paths: raise SystemExit(f"missing release outputs for {target}: {missing_paths}") output_dir.mkdir(parents=True, exist_ok=True) artifact_profile = SANDBOX_ARTIFACT_PROFILE if sandbox else RELEASE_ARTIFACT_PROFILE - staged_library = output_dir / staged_archive_name( - target, lib_path, artifact_profile - ) + staged_library = output_dir / staged_archive_name(target, lib_path, artifact_profile) staged_binding = output_dir / staged_binding_name(target, artifact_profile) with lib_path.open("rb") as src, staged_library.open("wb") as dst: @@ -259,9 +270,7 @@ def stage_artifacts( def upstream_release_pair_paths(source_root: Path, target: str) -> tuple[Path, Path]: - lib_name = ( - "rusty_v8.lib" if target.endswith("-pc-windows-msvc") else "librusty_v8.a" - ) + lib_name = "rusty_v8.lib" if target.endswith("-pc-windows-msvc") else "librusty_v8.a" gn_out = source_root / "target" / target / "release" / "gn_out" return gn_out / "obj" / lib_name, gn_out / "src_binding.rs" @@ -329,9 +338,7 @@ def parse_args() -> argparse.Namespace: stage_upstream_release_pair_parser = subparsers.add_parser( "stage-upstream-release-pair" ) - stage_upstream_release_pair_parser.add_argument( - "--source-root", type=Path, required=True - ) + stage_upstream_release_pair_parser.add_argument("--source-root", type=Path, required=True) stage_upstream_release_pair_parser.add_argument("--target", required=True) stage_upstream_release_pair_parser.add_argument("--output-dir", required=True) stage_upstream_release_pair_parser.add_argument("--sandbox", action="store_true") diff --git a/.github/scripts/test_run_bazel_with_buildbuddy.py b/.github/scripts/test_run_bazel_with_buildbuddy.py deleted file mode 100644 index 0f594b7947e..00000000000 --- a/.github/scripts/test_run_bazel_with_buildbuddy.py +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env python3 - -import json -import os -import subprocess -import sys -import unittest -from pathlib import Path -from tempfile import TemporaryDirectory - -import run_bazel_with_buildbuddy - - -class RunBazelWithBuildBuddyTest(unittest.TestCase): - def github_env( - self, - temp_dir: str, - *, - repository: str = "openai/codex", - fork: bool = False, - event_name: str = "pull_request", - ) -> dict[str, str]: - event_path = Path(temp_dir) / "event.json" - event_path.write_text( - json.dumps({"pull_request": {"head": {"repo": {"fork": fork}}}}), - encoding="utf-8", - ) - return { - "BUILDBUDDY_API_KEY": "token", - "GITHUB_ACTIONS": "true", - "GITHUB_EVENT_NAME": event_name, - "GITHUB_EVENT_PATH": str(event_path), - "GITHUB_REPOSITORY": repository, - } - - def test_keyless_invocation_drops_remote_ci_configuration(self) -> None: - self.assertIsNone( - run_bazel_with_buildbuddy.remote_config( - ["build", "--config=ci-linux", "//codex-rs/cli:codex"], - {}, - ) - ) - self.assertEqual( - run_bazel_with_buildbuddy.bazel_args_with_remote_config( - ["build", "--config=ci-linux", "--", "//codex-rs/cli:codex"], - {}, - ), - ["build", "--", "//codex-rs/cli:codex"], - ) - - def test_program_arguments_after_separator_do_not_select_or_lose_rbe(self) -> None: - args = ["run", "//codex-rs/cli:codex", "--", "--config=remote"] - - self.assertEqual( - run_bazel_with_buildbuddy.bazel_args_with_remote_config(args, {}), - args, - ) - self.assertEqual( - run_bazel_with_buildbuddy.remote_config( - args, {"BUILDBUDDY_API_KEY": "fork-token"} - ), - "buildbuddy-generic", - ) - - def test_upstream_push_selects_openai_rbe_before_target_separator(self) -> None: - with TemporaryDirectory() as temp_dir: - env = self.github_env(temp_dir, event_name="push") - - self.assertEqual( - run_bazel_with_buildbuddy.bazel_args_with_remote_config( - ["build", "--config=ci-linux", "--", "//codex-rs/cli:codex"], - env, - ), - [ - "build", - "--config=buildbuddy-openai-rbe", - "--remote_header=x-buildbuddy-api-key=token", - "--config=ci-linux", - "--", - "//codex-rs/cli:codex", - ], - ) - - def test_windows_cross_ci_configuration_follows_remote_configuration(self) -> None: - env = {"BUILDBUDDY_API_KEY": "fork-token"} - - self.assertEqual( - run_bazel_with_buildbuddy.bazel_args_with_remote_config( - ["build", "--config=ci-windows-cross", "//codex-rs/cli:codex"], - env, - ), - [ - "build", - "--config=buildbuddy-generic-rbe", - "--remote_header=x-buildbuddy-api-key=fork-token", - "--config=ci-windows-cross", - "//codex-rs/cli:codex", - ], - ) - - def test_query_remote_configuration_is_inserted_before_expression(self) -> None: - expression = 'kind("rust_library rule", //codex-rs/...)' - env = {"BUILDBUDDY_API_KEY": "fork-token"} - - for command in ("query", "cquery", "aquery"): - with self.subTest(command=command): - self.assertEqual( - run_bazel_with_buildbuddy.bazel_args_with_remote_config( - [ - command, - "--config=ci-windows-cross", - "--output=label", - expression, - ], - env, - ), - [ - command, - "--config=buildbuddy-generic-rbe", - "--remote_header=x-buildbuddy-api-key=fork-token", - "--config=ci-windows-cross", - "--output=label", - expression, - ], - ) - - def test_same_repository_pull_request_selects_openai_host(self) -> None: - with TemporaryDirectory() as temp_dir: - self.assertEqual( - run_bazel_with_buildbuddy.remote_config( - ["build", "--config=ci-v8"], self.github_env(temp_dir) - ), - "buildbuddy-openai-rbe", - ) - - def test_fork_pull_request_cannot_select_openai_host(self) -> None: - with TemporaryDirectory() as temp_dir: - env = self.github_env(temp_dir, fork=True) - - self.assertEqual( - run_bazel_with_buildbuddy.remote_config( - ["build", "--config=ci-v8"], env - ), - "buildbuddy-generic-rbe", - ) - - def test_run_in_fork_repository_cannot_select_openai_host(self) -> None: - with TemporaryDirectory() as temp_dir: - env = self.github_env(temp_dir, repository="contributor/codex") - - self.assertEqual( - run_bazel_with_buildbuddy.remote_config( - ["build", "--config=ci-v8"], env - ), - "buildbuddy-generic-rbe", - ) - - def test_pull_request_without_readable_event_payload_fails_closed(self) -> None: - for event_path in (None, "missing-event.json"): - env = { - "BUILDBUDDY_API_KEY": "token", - "GITHUB_ACTIONS": "true", - "GITHUB_EVENT_NAME": "pull_request", - "GITHUB_REPOSITORY": "openai/codex", - } - if event_path is not None: - env["GITHUB_EVENT_PATH"] = event_path - - with self.subTest(event_path=event_path): - self.assertEqual( - run_bazel_with_buildbuddy.remote_config(["build"], env), - "buildbuddy-generic", - ) - - def test_bazel_command_uses_configured_binary_locally(self) -> None: - self.assertEqual( - run_bazel_with_buildbuddy.bazel_command( - "info", - "execution_root", - env={"CODEX_BAZEL_BIN": "fake-bazel"}, - ), - ["fake-bazel", "info", "execution_root"], - ) - - def test_main_preserves_spaced_argument_and_child_exit_status(self) -> None: - spaced_arg = ( - r"--test_env=PATH=C:\Program Files\PowerShell\7;C:\Program Files\Git\bin" - ) - child_code = ( - f"import sys; sys.exit(37 if sys.argv[1] == {spaced_arg!r} else 91)" - ) - env = os.environ.copy() - env["CODEX_BAZEL_BIN"] = sys.executable - env.pop("BUILDBUDDY_API_KEY", None) - - result = subprocess.run( - [ - sys.executable, - str(Path(run_bazel_with_buildbuddy.__file__)), - "-c", - child_code, - spaced_arg, - ], - env=env, - check=False, - capture_output=True, - text=True, - ) - - self.assertEqual(result.returncode, 37, result.stderr) - - -if __name__ == "__main__": - unittest.main() diff --git a/.github/scripts/test_rusty_v8_bazel.py b/.github/scripts/test_rusty_v8_bazel.py index 0b5c03f4366..19690dbece4 100644 --- a/.github/scripts/test_rusty_v8_bazel.py +++ b/.github/scripts/test_rusty_v8_bazel.py @@ -88,49 +88,24 @@ def test_artifact_bazel_configs_always_enable_upstream_libcxx(self) -> None: ), ) - def test_bazel_commands_use_shared_buildbuddy_remote_config_library(self) -> None: - with patch.dict(environ, {}, clear=True): - self.assertEqual( - [ - "bazel", - "build", - "//third_party/v8:release", - ], - rusty_v8_bazel.bazel_command( - "build", - "--config=ci-v8", - "//third_party/v8:release", - ), - ) - with patch.dict(environ, {"BUILDBUDDY_API_KEY": "token"}, clear=True): + def test_bazel_remote_args_include_buildbuddy_header_when_present(self) -> None: + with patch.dict(environ, {"BUILDBUDDY_API_KEY": "token"}, clear=False): self.assertEqual( - [ - "bazel", - "build", - "--config=buildbuddy-generic-rbe", - "--remote_header=x-buildbuddy-api-key=token", - "--config=ci-v8", - "//third_party/v8:release", - ], - rusty_v8_bazel.bazel_command( - "build", - "--config=ci-v8", - "//third_party/v8:release", - ), + ["--remote_header=x-buildbuddy-api-key=token"], + rusty_v8_bazel.bazel_remote_args(), ) - def test_release_pair_labels_and_staged_names_distinguish_sandbox_artifacts( - self, - ) -> None: + with patch.dict(environ, {}, clear=True): + self.assertEqual([], rusty_v8_bazel.bazel_remote_args()) + + def test_release_pair_labels_and_staged_names_distinguish_sandbox_artifacts(self) -> None: self.assertEqual( "//third_party/v8:rusty_v8_release_pair_x86_64_unknown_linux_musl", rusty_v8_bazel.release_pair_label("x86_64-unknown-linux-musl"), ) self.assertEqual( "//third_party/v8:rusty_v8_sandbox_release_pair_x86_64_unknown_linux_musl", - rusty_v8_bazel.release_pair_label( - "x86_64-unknown-linux-musl", sandbox=True - ), + rusty_v8_bazel.release_pair_label("x86_64-unknown-linux-musl", sandbox=True), ) self.assertEqual( "//third_party/v8:rusty_v8_sandbox_release_pair_x86_64_apple_darwin", @@ -230,7 +205,11 @@ def test_stage_upstream_release_pair(self) -> None: with TemporaryDirectory() as source_dir, TemporaryDirectory() as output_dir: source_root = Path(source_dir) gn_out = ( - source_root / "target" / "x86_64-pc-windows-msvc" / "release" / "gn_out" + source_root + / "target" + / "x86_64-pc-windows-msvc" + / "release" + / "gn_out" ) (gn_out / "obj").mkdir(parents=True) (gn_out / "obj" / "rusty_v8.lib").write_bytes(b"archive") diff --git a/.github/termux-release.json b/.github/termux-release.json new file mode 100644 index 00000000000..6935021be7e --- /dev/null +++ b/.github/termux-release.json @@ -0,0 +1,15 @@ +{ + "upstream_repo": "openai/codex", + "upstream_tag": "rust-v0.138.0-alpha.4", + "upstream_name": "0.138.0-alpha.4", + "upstream_html_url": "https://github.com/openai/codex/releases/tag/rust-v0.138.0-alpha.4", + "upstream_target": "main", + "upstream_release_id": "334637864", + "upstream_prerelease": true, + "release_train": "0.138.0", + "release_branch": "release/0.138.0", + "work_branch": "upstream/rust-v0.138.0", + "patch_branch": "wallentx/termux-target", + "patch_source_sha": "c6e29647de1adcebb3f465f8e3d7f0c2d15cd118", + "termux_tag": "rust-v0.138.0-alpha.4-termux" +} diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index c6dfd60231d..11c0988ceb3 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -15,7 +15,6 @@ concurrency: # See https://docs.github.com/en/actions/using-jobs/using-concurrency and https://docs.github.com/en/actions/learn-github-actions/contexts for more info. group: concurrency-group::${{ github.workflow }}::${{ github.event.pull_request.number > 0 && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}${{ github.ref_name == 'main' && format('::{0}', github.run_id) || ''}} cancel-in-progress: ${{ github.ref_name != 'main' }} - jobs: test: # PRs use the sharded Windows cross-compiled test jobs below. Post-merge @@ -56,17 +55,12 @@ jobs: ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} persist-credentials: false - - uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2.62.49 - if: matrix.os == 'ubuntu-24.04' && matrix.target == 'x86_64-unknown-linux-gnu' - with: - tool: just - - name: Check rusty_v8 MODULE.bazel checksums if: matrix.os == 'ubuntu-24.04' && matrix.target == 'x86_64-unknown-linux-gnu' shell: bash run: | python3 .github/scripts/rusty_v8_bazel.py check-module-bazel - just test-github-scripts + python3 -m unittest discover -s .github/scripts -p test_rusty_v8_bazel.py - name: Prepare Bazel CI id: prepare_bazel @@ -147,9 +141,7 @@ jobs: - 2 - 3 - 4 - runs-on: - group: codex-runners - labels: codex-windows-x64 + runs-on: windows-latest name: Bazel test on windows-latest for x86_64-pc-windows-gnullvm shard ${{ matrix.shard }}/4 steps: @@ -158,11 +150,6 @@ jobs: ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} persist-credentials: false - - name: Test BuildBuddy Bazel wrapper - if: matrix.shard == 1 - shell: pwsh - run: python .github/scripts/test_run_bazel_with_buildbuddy.py - - name: Prepare Bazel CI id: prepare_bazel uses: ./.github/actions/prepare-bazel-ci @@ -259,9 +246,7 @@ jobs: # it a larger timeout. if: github.event_name == 'push' && github.ref == 'refs/heads/main' timeout-minutes: 40 - runs-on: - group: codex-runners - labels: codex-windows-x64 + runs-on: windows-latest name: Bazel test on windows-latest for x86_64-pc-windows-gnullvm (native main) steps: @@ -347,10 +332,7 @@ jobs: target: aarch64-apple-darwin - os: windows-latest target: x86_64-pc-windows-gnullvm - runs_on: - group: codex-runners - labels: codex-windows-x64 - runs-on: ${{ matrix.runs_on || matrix.os }} + runs-on: ${{ matrix.os }} name: Bazel clippy on ${{ matrix.os }} for ${{ matrix.target }} steps: @@ -440,10 +422,7 @@ jobs: target: aarch64-apple-darwin - os: windows-latest target: x86_64-pc-windows-gnullvm - runs_on: - group: codex-runners - labels: codex-windows-x64 - runs-on: ${{ matrix.runs_on || matrix.os }} + runs-on: ${{ matrix.os }} name: Verify release build on ${{ matrix.os }} for ${{ matrix.target }} steps: diff --git a/.github/workflows/cargo-deny.yml b/.github/workflows/cargo-deny.yml index bbadb57f943..f20d09e112e 100644 --- a/.github/workflows/cargo-deny.yml +++ b/.github/workflows/cargo-deny.yml @@ -6,11 +6,6 @@ on: branches: - main -# Cargo's libgit2 transport has been flaky when fetching git dependencies with -# nested submodules. Prefer the system git CLI across every Cargo invocation. -env: - CARGO_NET_GIT_FETCH_WITH_CLI: "true" - jobs: cargo-deny: runs-on: ubuntu-latest @@ -25,10 +20,10 @@ jobs: persist-credentials: false - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + uses: dtolnay/rust-toolchain@a0b273b48ed29de4470960879e8381ff45632f26 # 1.93.0 - name: Run cargo-deny uses: EmbarkStudios/cargo-deny-action@82eb9f621fbc699dd0918f3ea06864c14cc84246 # v2 with: - rust-version: 1.95.0 + rust-version: 1.93.0 manifest-path: ./codex-rs/Cargo.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63c6ffe5206..a1c60acc26d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,9 +26,6 @@ jobs: - name: Verify Bazel clippy flags match Cargo workspace lints run: python3 .github/scripts/verify_bazel_clippy_lints.py - - name: Test Codex package builder - run: python3 -m unittest discover -s scripts/codex_package -p 'test_*.py' - - name: Setup pnpm uses: pnpm/action-setup@a8198c4bff370c8506180b035930dea56dbd5288 # v5 with: @@ -42,6 +39,9 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile + # stage_npm_packages.py requires DotSlash when staging releases. + - uses: facebook/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2 + - name: Stage npm package id: stage_npm_package env: @@ -52,13 +52,15 @@ jobs: # cross-platform native payload required by the npm package layout. # Passing the workflow URL directly avoids relying on old rust-v* # branches remaining discoverable via `gh run list --branch ...`. - CODEX_VERSION=0.133.0-alpha.4 - WORKFLOW_URL="https://github.com/openai/codex/actions/runs/26201494185" + CODEX_VERSION=0.125.0 + WORKFLOW_URL="https://github.com/openai/codex/actions/runs/24901475298" OUTPUT_DIR="${RUNNER_TEMP}" + # This reused workflow predates the standalone bwrap artifact. python3 ./scripts/stage_npm_packages.py \ --release-version "$CODEX_VERSION" \ --workflow-url "$WORKFLOW_URL" \ --package codex \ + --allow-missing-native-component bwrap \ --output-dir "$OUTPUT_DIR" PACK_OUTPUT="${OUTPUT_DIR}/codex-npm-${CODEX_VERSION}.tgz" echo "pack_output=$PACK_OUTPUT" >> "$GITHUB_OUTPUT" @@ -74,15 +76,5 @@ jobs: - name: Check root README ToC run: python3 scripts/readme_toc.py README.md - - uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2.62.49 - with: - tool: just@1.51.0 - - name: Install uv - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - with: - version: "0.11.3" - - name: Check formatting (run `just fmt` to fix) - run: just fmt-check - - name: Prettier (run `pnpm run format:fix` to fix) run: pnpm run format diff --git a/.github/workflows/issue-deduplicator.yml b/.github/workflows/issue-deduplicator.yml index fea6348c46a..f15c1901025 100644 --- a/.github/workflows/issue-deduplicator.yml +++ b/.github/workflows/issue-deduplicator.yml @@ -12,7 +12,6 @@ jobs: # Prevent runs on forks (requires OpenAI API key, wastes Actions minutes) if: github.repository == 'openai/codex' && (github.event.action == 'opened' || (github.event.action == 'labeled' && github.event.label.name == 'codex-deduplicate')) runs-on: ubuntu-latest - environment: issue-triage permissions: contents: read outputs: @@ -158,7 +157,6 @@ jobs: needs: normalize-duplicates-all if: ${{ needs.normalize-duplicates-all.result == 'success' && needs.normalize-duplicates-all.outputs.has_matches != 'true' }} runs-on: ubuntu-latest - environment: issue-triage permissions: contents: read outputs: diff --git a/.github/workflows/issue-labeler.yml b/.github/workflows/issue-labeler.yml index 2c4eb6aa683..77fe5d07c88 100644 --- a/.github/workflows/issue-labeler.yml +++ b/.github/workflows/issue-labeler.yml @@ -12,7 +12,6 @@ jobs: # Prevent runs on forks (requires OpenAI API key, wastes Actions minutes) if: github.repository == 'openai/codex' && (github.event.action == 'opened' || (github.event.action == 'labeled' && github.event.label.name == 'codex-label')) runs-on: ubuntu-latest - environment: issue-triage permissions: contents: read outputs: diff --git a/.github/workflows/python-runtime-build.yml b/.github/workflows/python-runtime-build.yml deleted file mode 100644 index 1b91ab56927..00000000000 --- a/.github/workflows/python-runtime-build.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: python-runtime-build - -on: - workflow_call: - inputs: - runtime_version: - description: "Runtime version to build, for example 0.136.0 or 0.136.0a2." - required: true - type: string - -jobs: - build-python-runtime: - if: github.repository == 'openai/codex' - name: build-python-runtime - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Validate and resolve Python runtime release - id: python_runtime - shell: bash - env: - REQUESTED_RUNTIME_VERSION: ${{ inputs.runtime_version }} - run: | - set -euo pipefail - python3 - <<'PY' - import os - import re - from pathlib import Path - - python_version = os.environ["REQUESTED_RUNTIME_VERSION"] - if match := re.fullmatch(r"([0-9]+\.[0-9]+\.[0-9]+)a([0-9]+)", python_version): - release_version = f"{match.group(1)}-alpha.{match.group(2)}" - elif re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", python_version): - release_version = python_version - else: - raise SystemExit( - "Python runtime version must be stable or a numbered alpha, " - f"for example 0.136.0 or 0.136.0a2; found {python_version}" - ) - - with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: - print(f"python_version={python_version}", file=output) - print(f"release_tag=rust-v{release_version}", file=output) - PY - - - name: Download Python runtime release artifacts - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PYTHON_RUNTIME_VERSION: ${{ steps.python_runtime.outputs.python_version }} - RELEASE_TAG: ${{ steps.python_runtime.outputs.release_tag }} - run: | - set -euo pipefail - mkdir -p dist/python-runtime dist/python-runtime-packages - gh release download "$RELEASE_TAG" \ - --repo "${GITHUB_REPOSITORY}" \ - --pattern "openai_codex_cli_bin-${PYTHON_RUNTIME_VERSION}-*.whl" \ - --dir dist/python-runtime - gh release download "$RELEASE_TAG" \ - --repo "${GITHUB_REPOSITORY}" \ - --pattern "codex-package-*-unknown-linux-musl.tar.gz" \ - --dir dist/python-runtime-packages - - shopt -s nullglob - wheels=(dist/python-runtime/*.whl) - if [[ "${#wheels[@]}" -ne 6 ]]; then - echo "Expected 6 Python runtime wheels for ${PYTHON_RUNTIME_VERSION}, found ${#wheels[@]}." - exit 1 - fi - packages=(dist/python-runtime-packages/*.tar.gz) - if [[ "${#packages[@]}" -ne 2 ]]; then - echo "Expected 2 Linux package archives for ${PYTHON_RUNTIME_VERSION}, found ${#packages[@]}." - exit 1 - fi - - - name: Build musllinux Python runtime wheels - env: - RELEASE_TAG: ${{ steps.python_runtime.outputs.release_tag }} - run: | - set -euo pipefail - - python3 -m venv "${RUNNER_TEMP}/python-runtime-build-venv" - "${RUNNER_TEMP}/python-runtime-build-venv/bin/python" -m pip install build - - while read -r target platform_tag; do - stage_dir="${RUNNER_TEMP}/openai-codex-cli-bin-${target}-${platform_tag}" - python3 sdk/python/scripts/update_sdk_artifacts.py \ - stage-runtime \ - "$stage_dir" \ - "dist/python-runtime-packages/codex-package-${target}.tar.gz" \ - --codex-version "$RELEASE_TAG" \ - --platform-tag "$platform_tag" - "${RUNNER_TEMP}/python-runtime-build-venv/bin/python" -m build \ - --wheel \ - --outdir dist/python-runtime \ - "$stage_dir" - done <<'EOF' - aarch64-unknown-linux-musl musllinux_1_1_aarch64 - x86_64-unknown-linux-musl musllinux_1_1_x86_64 - EOF - - shopt -s nullglob - wheels=(dist/python-runtime/*.whl) - if [[ "${#wheels[@]}" -ne 8 ]]; then - echo "Expected 8 Python runtime wheels, found ${#wheels[@]}." - exit 1 - fi - ls -lh dist/python-runtime - - - name: Upload Python runtime wheels - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: python-runtime-wheels - path: dist/python-runtime/* - if-no-files-found: error diff --git a/.github/workflows/python-runtime-release.yml b/.github/workflows/python-runtime-release.yml deleted file mode 100644 index 4068786f319..00000000000 --- a/.github/workflows/python-runtime-release.yml +++ /dev/null @@ -1,100 +0,0 @@ -name: python-runtime-release - -on: - workflow_dispatch: - inputs: - runtime_version: - description: "Runtime version to publish before updating the SDK pin, for example 0.136.0 or 0.136.0a2." - required: true - type: string - -concurrency: - group: python-runtime-release-${{ inputs.runtime_version }} - cancel-in-progress: false - -jobs: - prepare-python-runtime: - name: prepare-python-runtime - permissions: - contents: read - uses: ./.github/workflows/python-runtime-build.yml - with: - runtime_version: ${{ inputs.runtime_version }} - - # PyPI must trust this top-level workflow for manual runtime publication. - publish-python-runtime: - if: github.repository == 'openai/codex' - name: publish-python-runtime - needs: prepare-python-runtime - runs-on: ubuntu-latest - environment: pypi - permissions: - contents: read - id-token: write # Required for PyPI trusted publishing. - - steps: - - name: Download Python runtime wheels - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: python-runtime-wheels - path: dist/python-runtime - - - name: Publish Python runtime wheels to PyPI - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 - with: - packages-dir: dist/python-runtime - skip-existing: true - - - name: Verify Python runtime wheels are available on PyPI - env: - PYTHON_RUNTIME_VERSION: ${{ inputs.runtime_version }} - run: | - set -euo pipefail - for attempt in {1..30}; do - if python3 - <<'PY' - import json - import os - import urllib.error - import urllib.request - - version = os.environ["PYTHON_RUNTIME_VERSION"] - tags = { - "macosx_10_9_x86_64", - "macosx_11_0_arm64", - "manylinux_2_17_aarch64", - "manylinux_2_17_x86_64", - "musllinux_1_1_aarch64", - "musllinux_1_1_x86_64", - "win_amd64", - "win_arm64", - } - expected = { - f"openai_codex_cli_bin-{version}-py3-none-{tag}.whl" - for tag in tags - } - - try: - with urllib.request.urlopen( - f"https://pypi.org/pypi/openai-codex-cli-bin/{version}/json", - timeout=30, - ) as response: - payload = json.load(response) - except urllib.error.URLError as error: - print(f"Could not read runtime {version} from PyPI: {error}.") - raise SystemExit(1) from error - - actual = {file["filename"] for file in payload["urls"]} - if actual != expected: - print(f"Missing runtime wheels: {sorted(expected - actual)}") - print(f"Unexpected runtime files: {sorted(actual - expected)}") - raise SystemExit(1) - PY - then - exit 0 - fi - echo "Runtime wheels are not available on PyPI yet; retrying (${attempt}/30)." - sleep 10 - done - - echo "Runtime wheels did not become available on PyPI." - exit 1 diff --git a/.github/workflows/python-sdk-release.yml b/.github/workflows/python-sdk-release.yml deleted file mode 100644 index 3ca930daa97..00000000000 --- a/.github/workflows/python-sdk-release.yml +++ /dev/null @@ -1,232 +0,0 @@ -name: python-sdk-release - -on: - push: - tags: - - "python-v*" - -concurrency: - group: ${{ github.workflow }} - cancel-in-progress: false - -jobs: - resolve-python-release: - if: github.repository == 'openai/codex' - name: resolve-python-release - runs-on: ubuntu-latest - permissions: - contents: read - outputs: - runtime_version: ${{ steps.python_release.outputs.runtime_version }} - sdk_version: ${{ steps.python_release.outputs.sdk_version }} - - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Validate SDK tag and resolve pinned runtime - id: python_release - shell: bash - run: | - set -euo pipefail - python3 - <<'PY' - import os - import re - import tomllib - from pathlib import Path - - sdk_version = os.environ["GITHUB_REF_NAME"].removeprefix("python-v") - if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+b[0-9]+", sdk_version): - raise SystemExit( - "Python SDK release tags must identify a beta release, " - "for example python-v0.1.0b1." - ) - - pyproject = tomllib.loads(Path("sdk/python/pyproject.toml").read_text()) - prefix = "openai-codex-cli-bin==" - runtime_versions = [ - dependency.removeprefix(prefix) - for dependency in pyproject["project"]["dependencies"] - if dependency.startswith(prefix) - ] - if len(runtime_versions) != 1: - raise SystemExit( - f"Expected exactly one pinned {prefix} dependency, found {runtime_versions}" - ) - - with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: - print(f"runtime_version={runtime_versions[0]}", file=output) - print(f"sdk_version={sdk_version}", file=output) - PY - - prepare-python-runtime: - name: prepare-python-runtime - needs: resolve-python-release - permissions: - contents: read - uses: ./.github/workflows/python-runtime-build.yml - with: - runtime_version: ${{ needs.resolve-python-release.outputs.runtime_version }} - - # Always publish the exact pinned runtime from this top-level workflow before - # building the SDK package. PyPI does not support reusable workflows as - # Trusted Publishers. - publish-python-runtime: - if: github.repository == 'openai/codex' - name: publish-python-runtime - needs: - - prepare-python-runtime - - resolve-python-release - runs-on: ubuntu-latest - environment: pypi - permissions: - contents: read - id-token: write # Required for PyPI trusted publishing. - - steps: - - name: Download Python runtime wheels - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: python-runtime-wheels - path: dist/python-runtime - - - name: Publish Python runtime wheels to PyPI - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 - with: - packages-dir: dist/python-runtime - skip-existing: true - - - name: Verify Python runtime wheels are available on PyPI - env: - PYTHON_RUNTIME_VERSION: ${{ needs.resolve-python-release.outputs.runtime_version }} - run: | - set -euo pipefail - for attempt in {1..30}; do - if python3 - <<'PY' - import json - import os - import urllib.error - import urllib.request - - version = os.environ["PYTHON_RUNTIME_VERSION"] - tags = { - "macosx_10_9_x86_64", - "macosx_11_0_arm64", - "manylinux_2_17_aarch64", - "manylinux_2_17_x86_64", - "musllinux_1_1_aarch64", - "musllinux_1_1_x86_64", - "win_amd64", - "win_arm64", - } - expected = { - f"openai_codex_cli_bin-{version}-py3-none-{tag}.whl" - for tag in tags - } - - try: - with urllib.request.urlopen( - f"https://pypi.org/pypi/openai-codex-cli-bin/{version}/json", - timeout=30, - ) as response: - payload = json.load(response) - except urllib.error.URLError as error: - print(f"Could not read runtime {version} from PyPI: {error}.") - raise SystemExit(1) from error - - actual = {file["filename"] for file in payload["urls"]} - if actual != expected: - print(f"Missing runtime wheels: {sorted(expected - actual)}") - print(f"Unexpected runtime files: {sorted(actual - expected)}") - raise SystemExit(1) - PY - then - exit 0 - fi - echo "Runtime wheels are not available on PyPI yet; retrying (${attempt}/30)." - sleep 10 - done - - echo "Runtime wheels did not become available on PyPI." - exit 1 - - build-python-sdk: - if: github.repository == 'openai/codex' - name: build-python-sdk - needs: - - publish-python-runtime - - resolve-python-release - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Build Python SDK package - shell: bash - env: - SDK_VERSION: ${{ needs.resolve-python-release.outputs.sdk_version }} - run: | - set -euo pipefail - - # Build in a glibc Linux image so release type generation installs - # the pinned manylinux runtime wheel. - docker run --rm \ - --user "$(id -u):$(id -g)" \ - -e HOME=/tmp/codex-python-sdk-home \ - -e UV_LINK_MODE=copy \ - -e SDK_VERSION \ - -e SDK_STAGE_DIR="${RUNNER_TEMP}/openai-codex" \ - -e SDK_DIST_DIR="${GITHUB_WORKSPACE}/dist/python-sdk" \ - -v "${GITHUB_WORKSPACE}:${GITHUB_WORKSPACE}" \ - -v "${RUNNER_TEMP}:${RUNNER_TEMP}" \ - -w "${GITHUB_WORKSPACE}/sdk/python" \ - python:3.12-slim \ - sh -euxc ' - python -m venv /tmp/release-tools - /tmp/release-tools/bin/python -m pip install build twine uv==0.11.3 - /tmp/release-tools/bin/uv sync --extra dev --frozen - /tmp/release-tools/bin/uv run --extra dev --frozen python scripts/update_sdk_artifacts.py \ - stage-sdk "${SDK_STAGE_DIR}" \ - --sdk-version "${SDK_VERSION}" - /tmp/release-tools/bin/python -m build \ - --wheel \ - --sdist \ - --outdir "${SDK_DIST_DIR}" \ - "${SDK_STAGE_DIR}" - /tmp/release-tools/bin/python -m twine check --strict "${SDK_DIST_DIR}/"* - ' - - - name: Upload Python SDK package - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: python-sdk-package - path: dist/python-sdk/* - if-no-files-found: error - - publish-python-sdk: - name: publish-python-sdk - needs: build-python-sdk - runs-on: ubuntu-latest - environment: pypi - permissions: - contents: read - id-token: write # Required for PyPI trusted publishing. - - steps: - - name: Download Python SDK package - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: python-sdk-package - path: dist/python-sdk - - - name: Publish Python SDK to PyPI - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 - with: - packages-dir: dist/python-sdk diff --git a/.github/workflows/rust-ci-full-nextest-platform.yml b/.github/workflows/rust-ci-full-nextest-platform.yml index 3fdf7b51eec..7dc39d33edf 100644 --- a/.github/workflows/rust-ci-full-nextest-platform.yml +++ b/.github/workflows/rust-ci-full-nextest-platform.yml @@ -94,7 +94,7 @@ jobs: - name: Install DotSlash uses: facebook/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2 - - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + - uses: dtolnay/rust-toolchain@a0b273b48ed29de4470960879e8381ff45632f26 # 1.93.0 with: targets: ${{ inputs.target }} @@ -319,7 +319,7 @@ jobs: - name: Install DotSlash uses: facebook/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2 - - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + - uses: dtolnay/rust-toolchain@a0b273b48ed29de4470960879e8381ff45632f26 # 1.93.0 with: targets: ${{ inputs.target }} diff --git a/.github/workflows/rust-ci-full.yml b/.github/workflows/rust-ci-full.yml index 60d508b21d7..08e0709e170 100644 --- a/.github/workflows/rust-ci-full.yml +++ b/.github/workflows/rust-ci-full.yml @@ -25,16 +25,11 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + - uses: dtolnay/rust-toolchain@a0b273b48ed29de4470960879e8381ff45632f26 # 1.93.0 with: components: rustfmt - - uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2.62.49 - with: - tool: just - name: cargo fmt run: cargo fmt -- --config imports_granularity=Item --check - - name: Rust benchmark smoke test - run: just bench-smoke cargo_shear: name: cargo shear @@ -46,7 +41,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + - uses: dtolnay/rust-toolchain@a0b273b48ed29de4470960879e8381ff45632f26 # 1.93.0 - uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2.62.49 with: tool: cargo-shear@1.11.2 @@ -63,7 +58,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + - uses: dtolnay/rust-toolchain@a0b273b48ed29de4470960879e8381ff45632f26 # 1.93.0 with: toolchain: nightly-2025-09-18 components: llvm-tools-preview, rustc-dev, rust-src @@ -260,9 +255,13 @@ jobs: set -euo pipefail if command -v apt-get >/dev/null 2>&1; then sudo apt-get update -y - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends pkg-config libcap-dev + packages=(pkg-config libcap-dev) + if [[ "${{ matrix.target }}" == 'x86_64-unknown-linux-musl' || "${{ matrix.target }}" == 'aarch64-unknown-linux-musl' ]]; then + packages+=(libubsan1) + fi + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "${packages[@]}" fi - - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + - uses: dtolnay/rust-toolchain@a0b273b48ed29de4470960879e8381ff45632f26 # 1.93.0 with: targets: ${{ matrix.target }} components: clippy @@ -344,6 +343,14 @@ jobs: sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}- sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}- + - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} + name: Disable sccache wrapper (musl) + shell: bash + run: | + set -euo pipefail + echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV" + echo "RUSTC_WORKSPACE_WRAPPER=" >> "$GITHUB_ENV" + - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} name: Prepare APT cache directories (musl) shell: bash @@ -377,9 +384,61 @@ jobs: shell: bash run: bash "${GITHUB_WORKSPACE}/.github/scripts/install-musl-build-tools.sh" - - if: ${{ !contains(matrix.target, 'windows') }} - name: Configure rusty_v8 artifact overrides and verify checksums - uses: ./.github/actions/setup-rusty-v8 + - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} + name: Configure rustc UBSan wrapper (musl host) + shell: bash + run: | + set -euo pipefail + ubsan="" + if command -v ldconfig >/dev/null 2>&1; then + ubsan="$(ldconfig -p | grep -m1 'libubsan\.so\.1' | sed -E 's/.*=> (.*)$/\1/')" + fi + wrapper_root="${RUNNER_TEMP:-/tmp}" + wrapper="${wrapper_root}/rustc-ubsan-wrapper" + cat > "${wrapper}" <> "$GITHUB_ENV" + echo "RUSTC_WORKSPACE_WRAPPER=" >> "$GITHUB_ENV" + + - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} + name: Clear sanitizer flags (musl) + shell: bash + run: | + set -euo pipefail + # Clear global Rust flags so host/proc-macro builds don't pull in UBSan. + echo "RUSTFLAGS=" >> "$GITHUB_ENV" + echo "CARGO_ENCODED_RUSTFLAGS=" >> "$GITHUB_ENV" + echo "RUSTDOCFLAGS=" >> "$GITHUB_ENV" + # Override any runner-level Cargo config rustflags as well. + echo "CARGO_BUILD_RUSTFLAGS=" >> "$GITHUB_ENV" + echo "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS=" >> "$GITHUB_ENV" + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS=" >> "$GITHUB_ENV" + echo "CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS=" >> "$GITHUB_ENV" + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS=" >> "$GITHUB_ENV" + + sanitize_flags() { + local input="$1" + input="${input//-fsanitize=undefined/}" + input="${input//-fno-sanitize-recover=undefined/}" + input="${input//-fno-sanitize-trap=undefined/}" + echo "$input" + } + + cflags="$(sanitize_flags "${CFLAGS-}")" + cxxflags="$(sanitize_flags "${CXXFLAGS-}")" + echo "CFLAGS=${cflags}" >> "$GITHUB_ENV" + echo "CXXFLAGS=${cxxflags}" >> "$GITHUB_ENV" + + - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl' }} + name: Configure musl rusty_v8 artifact overrides and verify checksums + uses: ./.github/actions/setup-rusty-v8-musl with: target: ${{ matrix.target }} diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 9b50ae40326..75c5c336012 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -3,11 +3,6 @@ on: pull_request: {} workflow_dispatch: -# Cargo's libgit2 transport has been flaky when fetching git dependencies with -# nested submodules. Prefer the system git CLI across every Cargo invocation. -env: - CARGO_NET_GIT_FETCH_WITH_CLI: "true" - jobs: # --- Detect what changed so the fast PR workflow only runs relevant jobs ---- changed: @@ -72,16 +67,11 @@ jobs: with: ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} persist-credentials: false - - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + - uses: dtolnay/rust-toolchain@a0b273b48ed29de4470960879e8381ff45632f26 # 1.93.0 with: components: rustfmt - - uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2.62.49 - with: - tool: just - name: cargo fmt run: cargo fmt -- --config imports_granularity=Item --check - - name: Rust benchmark smoke test - run: just bench-smoke cargo_shear: name: cargo shear @@ -96,7 +86,7 @@ jobs: with: ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} persist-credentials: false - - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + - uses: dtolnay/rust-toolchain@a0b273b48ed29de4470960879e8381ff45632f26 # 1.93.0 - uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2.62.49 with: tool: cargo-shear@1.11.2 @@ -116,7 +106,7 @@ jobs: with: ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} persist-credentials: false - - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + - uses: dtolnay/rust-toolchain@a0b273b48ed29de4470960879e8381ff45632f26 # 1.93.0 - name: Install nightly argument-comment-lint toolchain shell: bash run: | diff --git a/.github/workflows/rust-release-argument-comment-lint.yml b/.github/workflows/rust-release-argument-comment-lint.yml index 7f5ad01aa52..f654bd9dd72 100644 --- a/.github/workflows/rust-release-argument-comment-lint.yml +++ b/.github/workflows/rust-release-argument-comment-lint.yml @@ -7,11 +7,6 @@ on: required: true type: boolean -# Cargo's libgit2 transport has been flaky when fetching git dependencies with -# nested submodules. Prefer the system git CLI across every Cargo invocation. -env: - CARGO_NET_GIT_FETCH_WITH_CLI: "true" - jobs: skip: if: ${{ !inputs.publish }} @@ -65,7 +60,7 @@ jobs: with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + - uses: dtolnay/rust-toolchain@a0b273b48ed29de4470960879e8381ff45632f26 # 1.93.0 with: toolchain: nightly-2025-09-18 targets: ${{ matrix.target }} diff --git a/.github/workflows/rust-release-windows.yml b/.github/workflows/rust-release-windows.yml index 89bcd33ea1f..ac28b7855a1 100644 --- a/.github/workflows/rust-release-windows.yml +++ b/.github/workflows/rust-release-windows.yml @@ -20,11 +20,6 @@ on: AZURE_TRUSTED_SIGNING_CERTIFICATE_PROFILE_NAME: required: true -# Cargo's libgit2 transport has been flaky when fetching git dependencies with -# nested submodules. Prefer the system git CLI across every Cargo invocation. -env: - CARGO_NET_GIT_FETCH_WITH_CLI: "true" - jobs: build-windows-binaries: name: Build Windows binaries - ${{ matrix.runner }} - ${{ matrix.target }} - ${{ matrix.bundle }} @@ -105,22 +100,18 @@ jobs: Write-Host "Total RAM: $ramGiB GiB" Write-Host "Disk usage:" Get-PSDrive -PSProvider FileSystem | Format-Table -AutoSize Name, @{Name='Size(GB)';Expression={[math]::Round(($_.Used + $_.Free) / 1GB, 1)}}, @{Name='Free(GB)';Expression={[math]::Round($_.Free / 1GB, 1)}} - - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + - uses: dtolnay/rust-toolchain@a0b273b48ed29de4470960879e8381ff45632f26 # 1.93.0 with: targets: ${{ matrix.target }} - name: Cargo build (Windows binaries) shell: bash run: | - target="${{ matrix.target }}" - if [[ "$target" == "x86_64-pc-windows-msvc" ]]; then - export LIBSQLITE3_FLAGS=SQLITE_DISABLE_INTRINSIC - fi build_args=() for binary in ${{ matrix.binaries }}; do build_args+=(--bin "$binary") done - cargo build --target "$target" --release --timings "${build_args[@]}" + cargo build --target ${{ matrix.target }} --release --timings "${build_args[@]}" - name: Upload Cargo timings uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 @@ -229,9 +220,6 @@ jobs: "$dest/${binary}-${{ matrix.target }}.exe" done - - name: Install DotSlash - uses: facebook/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2 - - name: Build Codex package archives shell: bash run: | @@ -267,12 +255,16 @@ jobs: stage_dir="${RUNNER_TEMP}/openai-codex-cli-bin-${{ matrix.target }}" wheel_dir="${GITHUB_WORKSPACE}/python-runtime-dist/${{ matrix.target }}" + # Keep the helpers next to codex.exe in the runtime wheel so Windows + # sandbox/elevation lookup matches the standalone release zip. python "${GITHUB_WORKSPACE}/sdk/python/scripts/update_sdk_artifacts.py" \ stage-runtime \ "$stage_dir" \ - "dist/${{ matrix.target }}/codex-package-${{ matrix.target }}.tar.gz" \ + "${GITHUB_WORKSPACE}/codex-rs/target/${{ matrix.target }}/release/codex.exe" \ --codex-version "${GITHUB_REF_NAME}" \ - --platform-tag "$platform_tag" + --platform-tag "$platform_tag" \ + --resource-binary "${GITHUB_WORKSPACE}/codex-rs/target/${{ matrix.target }}/release/codex-command-runner.exe" \ + --resource-binary "${GITHUB_WORKSPACE}/codex-rs/target/${{ matrix.target }}/release/codex-windows-sandbox-setup.exe" "${RUNNER_TEMP}/python-runtime-build-venv/Scripts/python.exe" -m build --wheel --outdir "$wheel_dir" "$stage_dir" - name: Upload Python runtime wheel @@ -282,6 +274,9 @@ jobs: path: python-runtime-dist/${{ matrix.target }}/*.whl if-no-files-found: error + - name: Install DotSlash + uses: facebook/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2 + - name: Compress artifacts shell: bash run: | @@ -300,7 +295,7 @@ jobs: base="$(basename "$f")" # Skip files that are already archives (shouldn't happen, but be # safe). - if [[ "$base" == *.tar.gz || "$base" == *.tar.zst || "$base" == *.zip || "$base" == *.dmg ]]; then + if [[ "$base" == *.tar.gz || "$base" == *.zip || "$base" == *.dmg ]]; then continue fi diff --git a/.github/workflows/rust-release-zsh.yml b/.github/workflows/rust-release-zsh.yml index b55d2e714bc..492b8dc5e75 100644 --- a/.github/workflows/rust-release-zsh.yml +++ b/.github/workflows/rust-release-zsh.yml @@ -69,10 +69,6 @@ jobs: fail-fast: false matrix: include: - - runner: macos-15-large - target: x86_64-apple-darwin - variant: macos-15 - archive_name: codex-zsh-x86_64-apple-darwin.tar.gz - runner: macos-15-xlarge target: aarch64-apple-darwin variant: macos-15 diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index 5e3b263ed03..78162e86714 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -4,1423 +4,1028 @@ # git tag -a rust-v0.1.0 -m "Release 0.1.0" # git push origin rust-v0.1.0 # ``` -# -# Tag releases sign macOS binaries and DMGs through the protected `codesigning` -# GitHub environment and Azure Key Vault before final verification on macOS. -# -# To use external macOS signing, manually dispatch `release_mode=build_unsigned`, -# sign the unsigned macOS artifacts in a secure enclave, upload the signed handoff -# archive as a GitHub Release asset, then manually dispatch -# `release_mode=promote_signed` with `unsigned_run_id` and `signed_macos_asset`. -# The signed handoff archive should contain target or artifact directories such -# as `aarch64-apple-darwin/` with signed binaries. name: rust-release on: - push: - tags: - - "rust-v*.*.*" + pull_request: + branches: + - "release/**" workflow_dispatch: inputs: - release_mode: - description: "build_unsigned creates unsigned macOS handoff artifacts; promote_signed finishes a release from signed macOS handoff artifacts." + source_ref: + description: "Branch, tag, or commit SHA to build (code only; keeps .github from dispatched ref)" required: false + type: string + default: "" + build_target: + description: "Target to build" + required: true type: choice - default: build_unsigned + default: all options: - - build_unsigned - - promote_signed - sign_macos: - description: "Deprecated compatibility input; use release_mode instead." - required: false - type: boolean - default: false - unsigned_run_id: - description: "For promote_signed: workflow run id from the build_unsigned run." - required: false - type: string - signed_macos_asset: - description: "For promote_signed: exact GitHub Release asset name containing signed macOS handoff artifacts." - required: false - type: string - signed_macos_sha256: - description: "For promote_signed: optional SHA-256 of signed_macos_asset." - required: false - type: string - -concurrency: - group: ${{ github.workflow }} - cancel-in-progress: true + - all + - aarch64-apple-darwin + - x86_64-apple-darwin + - x86_64-unknown-linux-musl + - x86_64-unknown-linux-gnu + - aarch64-unknown-linux-musl + - aarch64-unknown-linux-gnu + - aarch64-linux-android + - x86_64-pc-windows-msvc + - aarch64-pc-windows-msvc + +permissions: + actions: read + attestations: read + checks: read + contents: read + deployments: read + issues: read + discussions: read + packages: read + pages: read + pull-requests: read + repository-projects: read + statuses: read jobs: + cancel-superseded-pr-runs: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + steps: + - name: Cancel older runs for this PR branch + env: + GH_TOKEN: ${{ github.token }} + HEAD_BRANCH: ${{ github.event.pull_request.head.ref }} + CURRENT_RUN_ID: ${{ github.run_id }} + shell: bash + run: | + set -euo pipefail + + echo "Cancelling older ${GITHUB_WORKFLOW} runs for ${HEAD_BRANCH}" + gh run list \ + --repo "${GITHUB_REPOSITORY}" \ + --workflow "${GITHUB_WORKFLOW}" \ + --limit 100 \ + --json databaseId,event,headBranch,status,url \ + --jq ' + .[] + | select(.event == "pull_request") + | select(.headBranch == env.HEAD_BRANCH) + | select(.databaseId < (env.CURRENT_RUN_ID | tonumber)) + | select(.status == "queued" or .status == "in_progress" or .status == "waiting" or .status == "requested" or .status == "pending") + | "\(.databaseId) \(.url)" + ' \ + | while read -r run_id run_url; do + [[ -n "${run_id}" ]] || continue + echo "Cancelling superseded run ${run_id}: ${run_url}" + gh run cancel "${run_id}" --repo "${GITHUB_REPOSITORY}" || true + done + tag-check: + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + - uses: actions/checkout@v6 + - name: 🧰 Actions Toolbox + # This is required for the GitHub CLI + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: wallentx/gh-actions/composite/actions-toolbox@main + - uses: dtolnay/rust-toolchain@1.92 - name: Validate tag matches Cargo.toml version shell: bash - env: - RELEASE_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.release_mode || 'signed' }} - REQUESTED_SIGN_MACOS: ${{ inputs.sign_macos }} - SIGNED_MACOS_ASSET: ${{ inputs.signed_macos_asset }} - UNSIGNED_RUN_ID: ${{ inputs.unsigned_run_id }} run: | set -euo pipefail echo "::group::Tag validation" - case "${RELEASE_MODE}" in - signed) - if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then - echo "❌ Manual rust-release runs must use release_mode=build_unsigned or release_mode=promote_signed" - exit 1 - fi - ;; - build_unsigned) - if [[ "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then - echo "❌ release_mode=build_unsigned is only valid for manual runs" - exit 1 - fi - ;; - promote_signed) - if [[ "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then - echo "❌ release_mode=promote_signed is only valid for manual runs" - exit 1 - fi - if [[ ! "${UNSIGNED_RUN_ID}" =~ ^[0-9]+$ ]]; then - echo "❌ release_mode=promote_signed requires unsigned_run_id to be a workflow run id" - exit 1 - fi - if [[ -z "${SIGNED_MACOS_ASSET}" ]]; then - echo "❌ release_mode=promote_signed requires signed_macos_asset" - exit 1 - fi - if [[ "${SIGNED_MACOS_ASSET}" == */* || "${SIGNED_MACOS_ASSET}" == *"*"* || "${SIGNED_MACOS_ASSET}" == *"?"* || "${SIGNED_MACOS_ASSET}" == *"["* ]]; then - echo "❌ signed_macos_asset must be an exact release asset name, not a path or glob" - exit 1 - fi - if [[ "${UNSIGNED_RUN_ID}" == "${GITHUB_RUN_ID}" ]]; then - echo "❌ unsigned_run_id must refer to the earlier build_unsigned run, not this run" - exit 1 - fi - ;; - *) - echo "❌ Unknown release_mode '${RELEASE_MODE}'" - exit 1 - ;; - esac - - if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" && "${REQUESTED_SIGN_MACOS}" == "true" ]]; then - echo "::warning title=Deprecated sign_macos input ignored::Use release_mode=build_unsigned or release_mode=promote_signed instead." - fi - - # All release modes must run from a tag. + # 1. Must be a tag and match the regex [[ "${GITHUB_REF_TYPE}" == "tag" ]] \ - || { echo "❌ Not a tag ref"; exit 1; } - - # Release tags must match the version in Cargo.toml. - [[ "${GITHUB_REF_NAME}" =~ ^rust-v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta)(\.[0-9]+)?)?$ ]] \ + || { echo "❌ Not a tag push"; exit 1; } + [[ "${GITHUB_REF_NAME}" =~ ^rust-v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta)(\.[0-9]+)?)?(-termux)?$ ]] \ || { echo "❌ Tag '${GITHUB_REF_NAME}' doesn't match expected format"; exit 1; } + # 2. Extract versions tag_ver="${GITHUB_REF_NAME#rust-v}" cargo_ver="$(grep -m1 '^version' codex-rs/Cargo.toml \ | sed -E 's/version *= *"([^"]+)".*/\1/')" + # 3. Compare [[ "${tag_ver}" == "${cargo_ver}" ]] \ || { echo "❌ Tag ${tag_ver} ≠ Cargo.toml ${cargo_ver}"; exit 1; } echo "✅ Tag and Cargo.toml agree (${tag_ver})" echo "::endgroup::" - build: - if: ${{ github.event_name != 'workflow_dispatch' || inputs.release_mode != 'promote_signed' }} + select-build-matrix: + if: always() && (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || needs.tag-check.result == 'success') needs: tag-check - name: Build - ${{ matrix.runner }} - ${{ matrix.target }} - ${{ matrix.bundle }} - runs-on: ${{ matrix.runs_on || matrix.runner }} - # Release builds can take a long time, so leave some headroom to avoid - # having to restart the full workflow due to a timeout. - timeout-minutes: 90 - permissions: - contents: read - id-token: write + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.select.outputs.matrix }} + build_version: ${{ steps.select.outputs.build_version }} + steps: + - uses: actions/checkout@v6 + - name: 🧰 Actions Toolbox + # This is required for the GitHub CLI + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: wallentx/gh-actions/composite/actions-toolbox@main + - id: select + shell: bash + env: + REPO_OWNER: ${{ github.repository_owner }} + EVENT_NAME: ${{ github.event_name }} + BUILD_TARGET: ${{ inputs.build_target }} + run: | + set -euo pipefail + + matrix='[ + {"runner":"macos-15-xlarge","target":"aarch64-apple-darwin"}, + {"runner":"macos-15-xlarge","target":"x86_64-apple-darwin"}, + {"runner":"ubuntu-24.04","target":"x86_64-unknown-linux-musl"}, + {"runner":"ubuntu-24.04","target":"x86_64-unknown-linux-gnu"}, + {"runner":"ubuntu-24.04-arm","target":"aarch64-unknown-linux-musl"}, + {"runner":"ubuntu-24.04-arm","target":"aarch64-unknown-linux-gnu"}, + {"runner":"ubuntu-24.04","target":"aarch64-linux-android"}, + {"runner":"windows-latest","target":"x86_64-pc-windows-msvc"}, + {"runner":"windows-11-arm","target":"aarch64-pc-windows-msvc"} + ]' + + if [[ "${REPO_OWNER}" != "openai" ]]; then + matrix="$(jq -c '[.[] | select(.runner | startswith("macos") | not)]' <<< "${matrix}")" + fi + + if [[ "${EVENT_NAME}" == "pull_request" ]]; then + matrix="$(jq -c '[.[] | select(.target == "aarch64-linux-android")]' <<< "${matrix}")" + elif [[ "${EVENT_NAME}" == "workflow_dispatch" && -n "${BUILD_TARGET}" && "${BUILD_TARGET}" != "all" ]]; then + matrix="$(jq -c --arg build_target "${BUILD_TARGET}" '[.[] | select(.target == $build_target)]' <<< "${matrix}")" + fi + + if [[ "$(jq 'length' <<< "${matrix}")" -eq 0 ]]; then + echo "No build targets selected after applying owner/dispatch filters." >&2 + exit 1 + fi + + echo "matrix={\"include\":${matrix}}" >> "$GITHUB_OUTPUT" + + if [[ "${EVENT_NAME}" == "pull_request" ]]; then + metadata=".github/termux-release.json" + if [[ ! -f "${metadata}" ]]; then + echo "${metadata} is required for release train PR builds" >&2 + exit 1 + fi + upstream_tag="$(jq -r '.upstream_tag // empty' "${metadata}")" + if [[ -z "${upstream_tag}" || "${upstream_tag}" != rust-v* ]]; then + echo "Unable to determine upstream rust tag from ${metadata}" >&2 + exit 1 + fi + build_version="${upstream_tag#rust-v}" + echo "build_version=${build_version}" >> "$GITHUB_OUTPUT" + elif [[ "${EVENT_NAME}" == "workflow_dispatch" ]]; then + latest_release_tag="$( + GH_TOKEN="${{ github.token }}" gh api repos/openai/codex/releases/latest --jq '.tag_name' + )" + + if [[ -z "${latest_release_tag}" || "${latest_release_tag}" != rust-v* ]]; then + echo "Unable to determine latest stable release tag from openai/codex" >&2 + exit 1 + fi + + latest_release_version="${latest_release_tag#rust-v}" + build_version="${latest_release_version}-dev" + echo "build_version=${build_version}" >> "$GITHUB_OUTPUT" + fi + + build: + if: >- + always() && + (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || needs.tag-check.result == 'success') && + (github.event_name != 'pull_request' || needs.cancel-superseded-pr-runs.result == 'success') && + needs.select-build-matrix.result == 'success' + needs: + - cancel-superseded-pr-runs + - tag-check + - select-build-matrix + name: Build - ${{ matrix.runner }} - ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + permissions: write-all + timeout-minutes: 120 defaults: run: working-directory: codex-rs env: - # 2026-03-04: temporarily change releases to use thin LTO because - # Ubuntu ARM is timing out at 60 minutes. - CARGO_PROFILE_RELEASE_LTO: ${{ contains(github.ref_name, '-alpha') && 'thin' || 'thin' }} - # Use the git CLI instead of Cargo's libgit2 path for git dependencies. - # macOS release runners have intermittently failed to fetch nested - # submodules through SecureTransport/libgit2, especially libwebrtc's - # libyuv submodule from chromium.googlesource.com. - CARGO_NET_GIT_FETCH_WITH_CLI: "true" + CODEX_BWRAP_ENABLE_FFI: ${{ contains(matrix.target, 'unknown-linux') && '1' || '0' }} + CODEX_BUILD_VERSION: ${{ needs.select-build-matrix.outputs.build_version }} + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" strategy: fail-fast: false - matrix: - include: - - runner: macos-15-xlarge - target: aarch64-apple-darwin - bundle: primary - artifact_name: aarch64-apple-darwin - binaries: "codex codex-responses-api-proxy" - build_dmg: "true" - - runner: macos-15-xlarge - target: aarch64-apple-darwin - bundle: app-server - artifact_name: aarch64-apple-darwin-app-server - binaries: "codex-app-server" - build_dmg: "false" - - runner: macos-15-xlarge - target: x86_64-apple-darwin - bundle: primary - artifact_name: x86_64-apple-darwin - binaries: "codex codex-responses-api-proxy" - build_dmg: "true" - - runner: macos-15-xlarge - target: x86_64-apple-darwin - bundle: app-server - artifact_name: x86_64-apple-darwin-app-server - binaries: "codex-app-server" - build_dmg: "false" - # Release artifacts intentionally ship MUSL-linked Linux binaries. - - runner: codex-linux-x64-xl - target: x86_64-unknown-linux-musl - bundle: primary - artifact_name: x86_64-unknown-linux-musl - binaries: "codex codex-responses-api-proxy bwrap" - build_dmg: "false" - - runner: codex-linux-x64-xl - target: x86_64-unknown-linux-musl - bundle: app-server - artifact_name: x86_64-unknown-linux-musl-app-server - binaries: "codex-app-server" - build_dmg: "false" - - runner: codex-linux-arm64 - target: aarch64-unknown-linux-musl - bundle: primary - artifact_name: aarch64-unknown-linux-musl - binaries: "codex codex-responses-api-proxy bwrap" - build_dmg: "false" - - runner: codex-linux-arm64 - target: aarch64-unknown-linux-musl - bundle: app-server - artifact_name: aarch64-unknown-linux-musl-app-server - binaries: "codex-app-server" - build_dmg: "false" + matrix: ${{ fromJson(needs.select-build-matrix.outputs.matrix) }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@v6 + - name: 🧰 Actions Toolbox + # This is required for the GitHub CLI + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: wallentx/gh-actions/composite/actions-toolbox@main with: - persist-credentials: false - - name: Print runner specs (Linux) - if: ${{ runner.os == 'Linux' }} - shell: bash - run: | - set -euo pipefail - cpu_model="$(lscpu | awk -F: '/Model name/ {gsub(/^[ \t]+/, "", $2); print $2; exit}')" - total_ram="$(awk '/MemTotal/ {printf "%.1f GiB\n", $2 / 1024 / 1024}' /proc/meminfo)" - echo "Runner: ${RUNNER_NAME:-unknown}" - echo "OS: $(uname -a)" - echo "CPU model: ${cpu_model}" - echo "Logical CPUs: $(nproc)" - echo "Total RAM: ${total_ram}" - echo "Disk usage:" - df -h . - - name: Print runner specs (macOS) - if: ${{ runner.os == 'macOS' }} - shell: bash - run: | - set -euo pipefail - total_ram="$(sysctl -n hw.memsize | awk '{printf "%.1f GiB\n", $1 / 1024 / 1024 / 1024}')" - echo "Runner: ${RUNNER_NAME:-unknown}" - echo "OS: $(sw_vers -productName) $(sw_vers -productVersion)" - echo "Hardware model: $(sysctl -n hw.model)" - echo "CPU architecture: $(uname -m)" - echo "Logical CPUs: $(sysctl -n hw.logicalcpu)" - echo "Physical CPUs: $(sysctl -n hw.physicalcpu)" - echo "Total RAM: ${total_ram}" - echo "Disk usage:" - df -h . - - name: Install Linux bwrap build dependencies - if: ${{ runner.os == 'Linux' }} - shell: bash - run: | - set -euo pipefail - sudo apt-get update -y - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends pkg-config libcap-dev - - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + verbose: true + - name: Checkout selected source ref + if: ${{ github.event_name == 'workflow_dispatch' && inputs.source_ref != '' }} + uses: actions/checkout@v6 with: - targets: ${{ matrix.target }} - - - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} - name: Use hermetic Cargo home (musl) + ref: ${{ inputs.source_ref }} + path: __source__ + fetch-depth: 1 + - name: Overlay selected source ref (excluding .github) + if: ${{ github.event_name == 'workflow_dispatch' && inputs.source_ref != '' }} shell: bash + working-directory: ${{ github.workspace }} run: | set -euo pipefail - cargo_home="${GITHUB_WORKSPACE}/.cargo-home" - mkdir -p "${cargo_home}/bin" - echo "CARGO_HOME=${cargo_home}" >> "$GITHUB_ENV" - echo "${cargo_home}/bin" >> "$GITHUB_PATH" - : > "${cargo_home}/config.toml" - - - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} - name: Install Zig - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 - with: - version: 0.14.0 - use-cache: false - - - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} - name: Install musl build tools - env: - TARGET: ${{ matrix.target }} - run: bash "${GITHUB_WORKSPACE}/.github/scripts/install-musl-build-tools.sh" - - - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} - name: Disable aws-lc jitter entropy (musl) + echo "Using source ref: ${{ inputs.source_ref }}" + + # Keep workflow/actions from the dispatch ref while replacing all other files. + find . -mindepth 1 -maxdepth 1 \ + ! -name .git \ + ! -name .github \ + ! -name __source__ \ + -exec rm -rf -- {} + + tar --exclude='.github' -C __source__ -cf - . | tar -xf - + rm -rf __source__ + + - name: Apply selected build version + if: ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }} shell: bash + working-directory: ${{ github.workspace }} run: | set -euo pipefail - # Avoid problematic aws-lc jitter entropy code path on musl builders. - echo "AWS_LC_SYS_NO_JITTER_ENTROPY=1" >> "$GITHUB_ENV" - target_no_jitter="AWS_LC_SYS_NO_JITTER_ENTROPY_${{ matrix.target }}" - target_no_jitter="${target_no_jitter//-/_}" - echo "${target_no_jitter}=1" >> "$GITHUB_ENV" - - - name: Configure rusty_v8 artifact overrides and verify checksums - uses: ./.github/actions/setup-rusty-v8 - with: - target: ${{ matrix.target }} - - - if: ${{ contains(matrix.target, 'linux') }} - name: Build bwrap and export digest - shell: bash - run: | - set -euo pipefail - target="${{ matrix.target }}" - cargo build --target "$target" --release --timings --bin bwrap - - bwrap_path="target/${target}/release/bwrap" - if [[ ! -f "$bwrap_path" ]]; then - echo "bwrap binary ${bwrap_path} not found" + if [[ -z "${CODEX_BUILD_VERSION}" ]]; then + echo "CODEX_BUILD_VERSION is empty for workflow_dispatch" >&2 exit 1 fi + echo "Using build version: ${CODEX_BUILD_VERSION}" + BUILD_VERSION="${CODEX_BUILD_VERSION}" perl -0777 -i -pe \ + 's/(\[workspace\.package\][^\[]*?version = ")([^"]+)(")/${1}$ENV{BUILD_VERSION}$3/s' \ + codex-rs/Cargo.toml - digest="$(sha256sum "$bwrap_path" | awk '{print $1}')" - echo "CODEX_BWRAP_SHA256=${digest}" >> "$GITHUB_ENV" - echo "Built bwrap ${bwrap_path} with sha256:${digest}" - - - name: Cargo build - shell: bash - run: | - target="${{ matrix.target }}" - if [[ "$target" == "x86_64-pc-windows-msvc" ]]; then - export LIBSQLITE3_FLAGS=SQLITE_DISABLE_INTRINSIC - fi - build_args=() - for binary in ${{ matrix.binaries }}; do - build_args+=(--bin "$binary") - done - echo "CARGO_PROFILE_RELEASE_LTO: ${CARGO_PROFILE_RELEASE_LTO}" - cargo build --target "$target" --release --timings "${build_args[@]}" - - - name: Upload Cargo timings - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: cargo-timings-rust-release-${{ matrix.target }}-${{ matrix.bundle }} - path: codex-rs/target/**/cargo-timings/cargo-timing.html - if-no-files-found: warn + grep -n "^\[workspace.package\]\|^version = " codex-rs/Cargo.toml | head -n 4 - - if: ${{ runner.os == 'macOS' }} - name: Stage unsigned macOS artifacts + - name: Select Android NDK + if: ${{ matrix.target == 'aarch64-linux-android' }} shell: bash run: | set -euo pipefail - target="${{ matrix.target }}" - release_dir="target/${target}/release" - dest="unsigned-dist/${target}" - mkdir -p "$dest" - - for binary in ${{ matrix.binaries }}; do - binary_path="${release_dir}/${binary}" - unsigned_name="${binary}-${target}-unsigned" - unsigned_path="${dest}/${unsigned_name}" - if [[ ! -f "${binary_path}" ]]; then - echo "Binary ${binary_path} not found" - exit 1 - fi - - cp "${binary_path}" "${unsigned_path}" - tar -C "$dest" -czf "${unsigned_path}.tar.gz" "${unsigned_name}" - zstd -T0 -19 --rm "${unsigned_path}" - done - - - if: ${{ runner.os == 'macOS' }} - name: Upload unsigned macOS artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: ${{ matrix.artifact_name }}-unsigned - path: codex-rs/unsigned-dist/${{ matrix.target }}/* - if-no-files-found: error - - - if: ${{ contains(matrix.target, 'linux') }} - name: Cosign Linux artifacts - uses: ./.github/actions/linux-code-sign - with: - target: ${{ matrix.target }} - artifacts-dir: ${{ github.workspace }}/codex-rs/target/${{ matrix.target }}/release - binaries: ${{ matrix.binaries }} - - - name: Stage artifacts - if: ${{ runner.os != 'macOS' }} - shell: bash - run: | - dest="dist/${{ matrix.target }}" - mkdir -p "$dest" + fallback_version="29.0.13113456" + ndk_root="${ANDROID_HOME}/ndk" + selected_ndk="" - for binary in ${{ matrix.binaries }}; do - cp "target/${{ matrix.target }}/release/${binary}" "$dest/${binary}-${{ matrix.target }}" - if [[ "${{ matrix.target }}" == *linux* ]]; then - cp "target/${{ matrix.target }}/release/${binary}.sigstore" \ - "$dest/${binary}-${{ matrix.target }}.sigstore" - fi - done + if [[ -d "${ndk_root}" ]]; then + while IFS= read -r candidate; do + toolchain="${candidate}/toolchains/llvm/prebuilt/linux-x86_64" + if [[ -x "${toolchain}/bin/aarch64-linux-android24-clang" ]]; then + selected_ndk="${candidate}" + break + fi + done < <(find "${ndk_root}" -mindepth 1 -maxdepth 1 -type d | sort -Vr) + fi - if [[ "${{ matrix.target }}" == *linux* && "${{ matrix.bundle }}" == "primary" ]]; then - bundle_root="${RUNNER_TEMP}/codex-${{ matrix.target }}-bundle" - rm -rf "$bundle_root" - mkdir -p "$bundle_root/codex-resources" - cp "$dest/codex-${{ matrix.target }}" "$bundle_root/codex" - cp "$dest/bwrap-${{ matrix.target }}" "$bundle_root/codex-resources/bwrap" - chmod 0755 "$bundle_root/codex" "$bundle_root/codex-resources/bwrap" - tar -C "$bundle_root" -cf - codex codex-resources/bwrap | - zstd -T0 -19 -o "$dest/codex-${{ matrix.target }}-bundle.tar.zst" + if [[ -z "${selected_ndk}" ]]; then + echo "No usable preinstalled Android NDK found; installing ${fallback_version}" + yes | sudo "${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" --licenses || true + sudo "${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" "ndk;${fallback_version}" + selected_ndk="${ndk_root}/${fallback_version}" fi - if [[ "${{ matrix.build_dmg }}" == "true" ]]; then - cp target/${{ matrix.target }}/release/codex-${{ matrix.target }}.dmg "$dest/codex-${{ matrix.target }}.dmg" + if [[ ! -x "${selected_ndk}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang" ]]; then + echo "Selected Android NDK does not contain a usable aarch64 API 24 clang: ${selected_ndk}" >&2 + exit 1 fi - - name: Build Codex package archive - if: ${{ runner.os != 'macOS' }} + echo "Selected Android NDK: ${selected_ndk}" + "${selected_ndk}/toolchains/llvm/prebuilt/linux-x86_64/bin/clang" --version + echo "ANDROID_NDK_HOME=${selected_ndk}" >> "$GITHUB_ENV" + echo "NDK_HOME=${selected_ndk}" >> "$GITHUB_ENV" + - name: Install Linux bwrap build dependencies + if: ${{ runner.os == 'Linux' && matrix.target != 'aarch64-linux-android' }} shell: bash - env: - TARGET: ${{ matrix.target }} - BUNDLE: ${{ matrix.bundle }} run: | set -euo pipefail - bash "${GITHUB_WORKSPACE}/.github/scripts/build-codex-package-archive.sh" \ - --target "$TARGET" \ - --bundle "$BUNDLE" \ - --entrypoint-dir "target/${TARGET}/release" \ - --archive-dir "dist/${TARGET}" - - - name: Build Python runtime wheel - if: ${{ matrix.bundle == 'primary' && runner.os != 'macOS' }} + sudo apt-get update -y + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends pkg-config libcap-dev + - name: Install UBSan runtime (musl) + if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl' }} shell: bash run: | set -euo pipefail - - case "${{ matrix.target }}" in - aarch64-apple-darwin) - platform_tag="macosx_11_0_arm64" - ;; - x86_64-apple-darwin) - platform_tag="macosx_10_9_x86_64" - ;; - aarch64-unknown-linux-musl) - platform_tag="manylinux_2_17_aarch64" - ;; - x86_64-unknown-linux-musl) - platform_tag="manylinux_2_17_x86_64" - ;; - *) - echo "No Python runtime wheel platform tag for ${{ matrix.target }}" - exit 1 - ;; - esac - - python3 -m venv "${RUNNER_TEMP}/python-runtime-build-venv" - # Do not install into the runner's system Python; macOS runners mark - # the Homebrew Python as externally managed under PEP 668. - "${RUNNER_TEMP}/python-runtime-build-venv/bin/python" -m pip install build - - stage_dir="${RUNNER_TEMP}/openai-codex-cli-bin-${{ matrix.target }}" - wheel_dir="${GITHUB_WORKSPACE}/python-runtime-dist/${{ matrix.target }}" - stage_runtime_args=( - "${GITHUB_WORKSPACE}/sdk/python/scripts/update_sdk_artifacts.py" - stage-runtime - "$stage_dir" - "dist/${{ matrix.target }}/codex-package-${{ matrix.target }}.tar.gz" - --codex-version "${GITHUB_REF_NAME}" - --platform-tag "$platform_tag" - ) - python3 "${stage_runtime_args[@]}" - "${RUNNER_TEMP}/python-runtime-build-venv/bin/python" -m build --wheel --outdir "$wheel_dir" "$stage_dir" - - - name: Upload Python runtime wheel - if: ${{ matrix.bundle == 'primary' && runner.os != 'macOS' }} - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: python-runtime-wheel-${{ matrix.target }} - path: python-runtime-dist/${{ matrix.target }}/*.whl - if-no-files-found: error - - - name: Compress artifacts - if: ${{ runner.os != 'macOS' }} - shell: bash - run: | - # Path that contains the uncompressed binaries for the current - # ${{ matrix.target }} - dest="dist/${{ matrix.target }}" - - # For compatibility with environments that lack the `zstd` tool we - # additionally create a `.tar.gz` alongside every binary we publish. - # The end result is: - # codex-.zst (existing) - # codex-.tar.gz (new) - - # 1. Produce a .tar.gz for every file in the directory *before* we - # run `zstd --rm`, because that flag deletes the original files. - for f in "$dest"/*; do - base="$(basename "$f")" - # Skip files that are already archives (shouldn't happen, but be - # safe). - if [[ "$base" == *.tar.gz || "$base" == *.tar.zst || "$base" == *.zip || "$base" == *.dmg ]]; then - continue - fi - - # Don't try to compress signature bundles. - if [[ "$base" == *.sigstore ]]; then - continue - fi - - # Create per-binary tar.gz - tar -C "$dest" -czf "$dest/${base}.tar.gz" "$base" - - # Also create .zst and remove the uncompressed binaries to keep - # non-Windows artifact directories small. - zstd -T0 -19 --rm "$dest/$base" - done - - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - if: ${{ runner.os != 'macOS' }} - with: - name: ${{ matrix.artifact_name }} - # Upload the per-binary .zst files, .tar.gz equivalents, and any - # prebuilt archives staged above. - path: | - codex-rs/dist/${{ matrix.target }}/* - - sign-macos-binaries: - if: ${{ github.event_name != 'workflow_dispatch' }} - needs: build - name: Sign macOS binaries - ${{ matrix.target }} - ${{ matrix.bundle }} - runs-on: ubuntu-latest - timeout-minutes: 45 - environment: - name: codesigning - deployment: false - permissions: - contents: read - id-token: write - - strategy: - fail-fast: false - matrix: - include: - - target: aarch64-apple-darwin - bundle: primary - artifact_name: aarch64-apple-darwin - binaries: "codex codex-responses-api-proxy" - - target: aarch64-apple-darwin - bundle: app-server - artifact_name: aarch64-apple-darwin-app-server - binaries: "codex-app-server" - - target: x86_64-apple-darwin - bundle: primary - artifact_name: x86_64-apple-darwin - binaries: "codex codex-responses-api-proxy" - - target: x86_64-apple-darwin - bundle: app-server - artifact_name: x86_64-apple-darwin-app-server - binaries: "codex-app-server" - - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Download unsigned macOS binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: ${{ matrix.artifact_name }}-unsigned - path: ${{ runner.temp }}/unsigned-macos - - - name: Set up AKV PKCS11 macOS signing - uses: ./.github/actions/setup-akv-pkcs11-codesigning + if command -v apt-get >/dev/null 2>&1; then + sudo apt-get update -y + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libubsan1 + fi + - uses: dtolnay/rust-toolchain@1.93 with: - rcodesign-blob-uri: ${{ secrets.AKV_CODESIGN_RCODESIGN_BLOB_URI }} - rcodesign-sha256: ${{ secrets.AKV_CODESIGN_RCODESIGN_SHA256 }} - akv-pkcs11-library-blob-uri: ${{ secrets.AKV_CODESIGN_PKCS11_LIBRARY_BLOB_URI }} - akv-pkcs11-library-sha256: ${{ secrets.AKV_CODESIGN_PKCS11_LIBRARY_SHA256 }} - azure-client-id: ${{ secrets.AKV_CODESIGN_AZURE_CLIENT_ID }} - azure-tenant-id: ${{ secrets.AKV_CODESIGN_TENANT }} - azure-subscription-id: ${{ secrets.AKV_CODESIGN_SUBSCRIPTION }} - key-vault-name: ${{ secrets.AKV_CODESIGN_KEY_VAULT_NAME }} - key-name: ${{ secrets.AKV_CODESIGN_KEY_NAME }} - key-version: ${{ secrets.AKV_CODESIGN_KEY_VERSION || '' }} - certificate-sha256: ${{ secrets.AKV_CODESIGN_CERTIFICATE_SHA256 || '' }} - - - name: Sign and notarize macOS binaries + targets: ${{ matrix.target }} + - name: Set up sccache (Android) + if: ${{ matrix.target == 'aarch64-linux-android' }} + uses: mozilla-actions/sccache-action@main + - name: Enable sccache (Android) + if: ${{ matrix.target == 'aarch64-linux-android' }} shell: bash - env: - TARGET: ${{ matrix.target }} - BINARIES: ${{ matrix.binaries }} - APPLE_NOTARIZATION_KEY_P8: ${{ secrets.APPLE_NOTARIZATION_KEY_P8 }} - APPLE_NOTARIZATION_KEY_ID: ${{ secrets.APPLE_NOTARIZATION_KEY_ID }} - APPLE_NOTARIZATION_ISSUER_ID: ${{ secrets.APPLE_NOTARIZATION_ISSUER_ID }} run: | set -euo pipefail - - input_dir="${RUNNER_TEMP}/unsigned-macos" - output_dir="${GITHUB_WORKSPACE}/signed-macos/${TARGET}" - report_dir="${GITHUB_WORKSPACE}/macos-binary-signing-verification/${TARGET}" - mkdir -p "$output_dir" "$report_dir" - - for binary in ${BINARIES}; do - unsigned_path="${input_dir}/${binary}-${TARGET}-unsigned.zst" - signed_path="${output_dir}/${binary}" - if [[ ! -f "$unsigned_path" ]]; then - echo "Unsigned binary $unsigned_path not found" - exit 1 - fi - - zstd -d --stdout "$unsigned_path" >"$signed_path" - chmod 0755 "$signed_path" - - .github/scripts/macos-signing/sign_macos_code.sh \ - --target "$signed_path" \ - --identity unused \ - --deep false \ - --identifier "$binary" \ - --options runtime \ - --timestamp true \ - --entitlements .github/scripts/macos-signing/codex.entitlements.plist - - mkdir -p "${report_dir}/${binary}" - rcodesign print-signature-info "$signed_path" \ - >"${report_dir}/${binary}/signature-info.yaml" - - .github/scripts/macos-signing/notarize_macos_binary_with_rcodesign.sh \ - --binary "$signed_path" \ - --report-dir "${report_dir}/${binary}" - done - - - name: Upload signed macOS binaries - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: ${{ matrix.artifact_name }}-signed-binaries - path: signed-macos/${{ matrix.target }}/* - if-no-files-found: error - - - name: Upload binary signing verification - if: ${{ always() }} - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: ${{ matrix.artifact_name }}-binary-signing-verification - path: macos-binary-signing-verification/${{ matrix.target }}/ - if-no-files-found: warn - - package-macos: - if: ${{ github.event_name != 'workflow_dispatch' }} - needs: sign-macos-binaries - name: Package macOS artifacts - ${{ matrix.target }} - ${{ matrix.bundle }} - runs-on: macos-15-xlarge - timeout-minutes: 45 - permissions: - contents: read - defaults: - run: - working-directory: codex-rs - - strategy: - fail-fast: false - matrix: - include: - - target: aarch64-apple-darwin - bundle: primary - artifact_name: aarch64-apple-darwin - binaries: "codex codex-responses-api-proxy" - build_dmg: "true" - - target: aarch64-apple-darwin - bundle: app-server - artifact_name: aarch64-apple-darwin-app-server - binaries: "codex-app-server" - build_dmg: "false" - - target: x86_64-apple-darwin - bundle: primary - artifact_name: x86_64-apple-darwin - binaries: "codex codex-responses-api-proxy" - build_dmg: "true" - - target: x86_64-apple-darwin - bundle: app-server - artifact_name: x86_64-apple-darwin-app-server - binaries: "codex-app-server" - build_dmg: "false" - - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Download signed macOS binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: ${{ matrix.artifact_name }}-signed-binaries - path: codex-rs/target/${{ matrix.target }}/release - - - name: Verify signed macOS binaries + echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" + echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" + echo "SCCACHE_GHA_VERSION=android-release-${{ matrix.target }}" >> "$GITHUB_ENV" + echo "SCCACHE_CACHE_SIZE=5G" >> "$GITHUB_ENV" + - name: Ensure Rust target is installed shell: bash run: | set -euo pipefail - for binary in ${{ matrix.binaries }}; do - binary_path="target/${{ matrix.target }}/release/${binary}" - chmod 0755 "$binary_path" - codesign --verify --strict --verbose=2 "$binary_path" - done - - - name: Build unsigned macOS DMG - if: ${{ matrix.build_dmg == 'true' }} + rustup target add "${{ matrix.target }}" + - if: ${{ matrix.target == 'aarch64-linux-android' }} + name: Configure Android build environment shell: bash run: | set -euo pipefail - + ndk="${ANDROID_NDK_HOME}" target="${{ matrix.target }}" - release_dir="target/${target}/release" - dmg_root="${RUNNER_TEMP}/codex-dmg-root-${target}" - volname="Codex (${target})" - dmg_path="${release_dir}/codex-${target}.dmg" - rm -rf "$dmg_root" - mkdir -p "$dmg_root" - - for binary in ${{ matrix.binaries }}; do - binary_path="${release_dir}/${binary}" - if [[ ! -f "$binary_path" ]]; then - echo "Binary $binary_path not found" - exit 1 - fi - ditto "$binary_path" "${dmg_root}/${binary}" - done + # Set up the Android toolchain + toolchain="${ndk}/toolchains/llvm/prebuilt/linux-x86_64" + + # Use API level 24 to follow termux-build convention + api_level="24" + + # Configure Cargo to use the NDK linker + cargo_target_var="CARGO_TARGET_${target^^}_LINKER" + cargo_target_var="${cargo_target_var//-/_}" + echo "${cargo_target_var}=${toolchain}/bin/aarch64-linux-android${api_level}-clang" >> "$GITHUB_ENV" + + # Set CC and AR for the target + target_cc_var="CC_${target}" + target_cc_var="${target_cc_var//-/_}" + echo "${target_cc_var}=${toolchain}/bin/aarch64-linux-android${api_level}-clang" >> "$GITHUB_ENV" + + target_cxx_var="CXX_${target}" + target_cxx_var="${target_cxx_var//-/_}" + echo "${target_cxx_var}=${toolchain}/bin/aarch64-linux-android${api_level}-clang++" >> "$GITHUB_ENV" + + target_ar_var="AR_${target}" + target_ar_var="${target_ar_var//-/_}" + echo "${target_ar_var}=${toolchain}/bin/llvm-ar" >> "$GITHUB_ENV" + + # Add toolchain to PATH + echo "${toolchain}/bin" >> "$GITHUB_PATH" + + tls_align_source="${RUNNER_TEMP}/codex-android-tls-align.S" + tls_align_object="${RUNNER_TEMP}/codex-android-tls-align.o" + cat > "${tls_align_source}" <<'EOF' + .section .tdata.codex_android_tls_align,"awT",%progbits + .p2align 6 + .globl __codex_android_tls_align + .hidden __codex_android_tls_align + .type __codex_android_tls_align, %object + __codex_android_tls_align: + .byte 0 + .size __codex_android_tls_align, 1 + EOF + "${toolchain}/bin/aarch64-linux-android${api_level}-clang" -c "${tls_align_source}" -o "${tls_align_object}" + + v8_compat_source="${RUNNER_TEMP}/codex-android-v8-compat.c" + v8_compat_object="${RUNNER_TEMP}/codex-android-v8-compat.o" + cat > "${v8_compat_source}" <<'EOF' + #include + #include + + extern int posix_memalign(void **memptr, size_t alignment, size_t size); + extern double strtod(const char *nptr, char **endptr); + extern float strtof(const char *nptr, char **endptr); + + void *aligned_alloc(size_t alignment, size_t size) { + void *ptr = 0; + if (posix_memalign(&ptr, alignment, size) != 0) { + return 0; + } + return ptr; + } - rm -f "$dmg_path" - hdiutil create \ - -volname "$volname" \ - -srcfolder "$dmg_root" \ - -format UDZO \ - -ov \ - "$dmg_path" + double strtod_l(const char *nptr, char **endptr, locale_t locale) { + (void)locale; + return strtod(nptr, endptr); + } - if [[ ! -f "$dmg_path" ]]; then - echo "DMG $dmg_path not found after build" + float strtof_l(const char *nptr, char **endptr, locale_t locale) { + (void)locale; + return strtof(nptr, endptr); + } + EOF + "${toolchain}/bin/aarch64-linux-android${api_level}-clang" -c "${v8_compat_source}" -o "${v8_compat_object}" + + builtins_archive="$( + find "${toolchain}/lib/clang" \ + -path "*/lib/linux/libclang_rt.builtins-aarch64-android.a" \ + -print -quit + )" + if [[ -z "${builtins_archive}" ]]; then + echo "Could not find Android compiler-rt builtins archive under ${toolchain}/lib/clang" >&2 exit 1 fi + builtins_dir="$(dirname "${builtins_archive}")" - - name: Upload unsigned macOS DMG - if: ${{ matrix.build_dmg == 'true' }} - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: ${{ matrix.artifact_name }}-unsigned-dmg - path: codex-rs/target/${{ matrix.target }}/release/codex-${{ matrix.target }}.dmg - if-no-files-found: error + # Provide compatibility symbols and C++ ABI needed by the Android rusty_v8 archive. + rustflags_var="CARGO_TARGET_${target^^}_RUSTFLAGS" + rustflags_var="${rustflags_var//-/_}" + echo "${rustflags_var}=-C link-arg=${tls_align_object} -C link-arg=${v8_compat_object} -C link-arg=-Wl,-u,__codex_android_tls_align -C link-arg=-L${builtins_dir} -C link-arg=-lclang_rt.builtins-aarch64-android -C link-arg=-lc++abi -C link-arg=-Wl,-z,max-page-size=65536" >> "$GITHUB_ENV" - - name: Stage macOS artifacts - shell: bash - run: | - set -euo pipefail - dest="dist/${{ matrix.target }}" - mkdir -p "$dest" - for binary in ${{ matrix.binaries }}; do - cp "target/${{ matrix.target }}/release/${binary}" "$dest/${binary}-${{ matrix.target }}" - done + # Configure for vendored OpenSSL build + echo "CARGO_BUILD_TARGET_APPLIES_TO_HOST=false" >> "$GITHUB_ENV" + echo "CARGO_BUILD_TARGET=${{ matrix.target }}" >> "$GITHUB_ENV" - - name: Build Codex package archive - shell: bash + - if: ${{ matrix.target == 'aarch64-linux-android' }} + name: Configure Android rusty_v8 artifact overrides env: + GH_TOKEN: ${{ github.token }} TARGET: ${{ matrix.target }} - BUNDLE: ${{ matrix.bundle }} + shell: bash run: | set -euo pipefail - bash "${GITHUB_WORKSPACE}/.github/scripts/build-codex-package-archive.sh" \ - --target "$TARGET" \ - --bundle "$BUNDLE" \ - --entrypoint-dir "target/${TARGET}/release" \ - --archive-dir "dist/${TARGET}" - - - name: Build Python runtime wheel - if: ${{ matrix.bundle == 'primary' }} + binding_dir="${RUNNER_TEMP}/rusty_v8" + archive="${binding_dir}/librusty_v8_release_${TARGET}.a.gz" + binding_path="${binding_dir}/src_binding_release_${TARGET}.rs" + checksums_path="${binding_dir}/rusty_v8_release_${TARGET}.sha256" + artifact_repository="wallentx/codex-termux" + release_tag="rusty-v8-v147.4.0" + mkdir -p "${binding_dir}" + echo "Downloading Android rusty_v8 artifacts from ${artifact_repository}@${release_tag}" + gh release download "${release_tag}" \ + --repo "${artifact_repository}" \ + --dir "${binding_dir}" \ + --pattern "librusty_v8_release_${TARGET}.a.gz" \ + --pattern "src_binding_release_${TARGET}.rs" \ + --pattern "rusty_v8_release_${TARGET}.sha256" + (cd "${binding_dir}" && sha256sum -c "$(basename "${checksums_path}")") + echo "RUSTY_V8_ARCHIVE=${archive}" >> "$GITHUB_ENV" + echo "RUSTY_V8_SRC_BINDING_PATH=${binding_path}" >> "$GITHUB_ENV" + + - if: ${{ matrix.target == 'aarch64-linux-android' }} + name: Install termux-elf-cleaner shell: bash run: | set -euo pipefail + version="v3.0.1" + src_dir="${RUNNER_TEMP}/termux-elf-cleaner-src" + build_dir="${RUNNER_TEMP}/termux-elf-cleaner-build" + bin_dir="${RUNNER_TEMP}/termux-elf-cleaner-bin" - case "${{ matrix.target }}" in - aarch64-apple-darwin) - platform_tag="macosx_11_0_arm64" - ;; - x86_64-apple-darwin) - platform_tag="macosx_10_9_x86_64" - ;; - *) - echo "No Python runtime wheel platform tag for ${{ matrix.target }}" - exit 1 - ;; - esac - - python3 -m venv "${RUNNER_TEMP}/python-runtime-build-venv" - "${RUNNER_TEMP}/python-runtime-build-venv/bin/python" -m pip install build - - stage_dir="${RUNNER_TEMP}/openai-codex-cli-bin-${{ matrix.target }}" - wheel_dir="${GITHUB_WORKSPACE}/python-runtime-dist/${{ matrix.target }}" - python3 \ - "${GITHUB_WORKSPACE}/sdk/python/scripts/update_sdk_artifacts.py" \ - stage-runtime \ - "$stage_dir" \ - "dist/${{ matrix.target }}/codex-package-${{ matrix.target }}.tar.gz" \ - --codex-version "${GITHUB_REF_NAME}" \ - --platform-tag "$platform_tag" - "${RUNNER_TEMP}/python-runtime-build-venv/bin/python" -m build --wheel --outdir "$wheel_dir" "$stage_dir" - - - name: Upload Python runtime wheel - if: ${{ matrix.bundle == 'primary' }} - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: python-runtime-wheel-${{ matrix.target }} - path: python-runtime-dist/${{ matrix.target }}/*.whl - if-no-files-found: error + rm -rf "${src_dir}" "${build_dir}" "${bin_dir}" + mkdir -p "${src_dir}" "${build_dir}" "${bin_dir}" - - name: Compress artifacts - shell: bash - run: | - set -euo pipefail - dest="dist/${{ matrix.target }}" - for f in "$dest"/*; do - base="$(basename "$f")" - if [[ "$base" == *.tar.gz || "$base" == *.tar.zst || "$base" == *.zip || "$base" == *.dmg ]]; then - continue - fi + curl -fsSL "https://github.com/termux/termux-elf-cleaner/archive/refs/tags/${version}.tar.gz" \ + | tar -xzf - --strip-components=1 -C "${src_dir}" - tar -C "$dest" -czf "$dest/${base}.tar.gz" "$base" - zstd -T0 -19 --rm "$dest/$base" - done + cmake -S "${src_dir}" -B "${build_dir}" -DCMAKE_BUILD_TYPE=Release + cmake --build "${build_dir}" --parallel - - name: Upload packaged macOS artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: ${{ matrix.artifact_name }}-packaged - path: codex-rs/dist/${{ matrix.target }}/* - if-no-files-found: error - - sign-macos-dmg: - if: ${{ github.event_name != 'workflow_dispatch' }} - needs: package-macos - name: Sign macOS DMG - ${{ matrix.target }} - runs-on: ubuntu-latest - timeout-minutes: 45 - environment: - name: codesigning - deployment: false - permissions: - contents: read - id-token: write + install "${build_dir}/termux-elf-cleaner" "${bin_dir}/termux-elf-cleaner" + echo "${bin_dir}" >> "$GITHUB_PATH" - strategy: - fail-fast: false - matrix: - include: - - target: aarch64-apple-darwin - artifact_name: aarch64-apple-darwin - - target: x86_64-apple-darwin - artifact_name: x86_64-apple-darwin + - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} + name: Use hermetic Cargo home (musl) + shell: bash + run: | + set -euo pipefail + cargo_home="${GITHUB_WORKSPACE}/.cargo-home" + mkdir -p "${cargo_home}/bin" + echo "CARGO_HOME=${cargo_home}" >> "$GITHUB_ENV" + echo "${cargo_home}/bin" >> "$GITHUB_PATH" + : > "${cargo_home}/config.toml" - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/cache@v5 with: - persist-credentials: false + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ${{ github.workspace }}/.cargo-home/bin/ + ${{ github.workspace }}/.cargo-home/registry/index/ + ${{ github.workspace }}/.cargo-home/registry/cache/ + ${{ github.workspace }}/.cargo-home/git/db/ + ${{ github.workspace }}/codex-rs/target/ + key: cargo-${{ matrix.runner }}-${{ matrix.target }}-release-${{ hashFiles('**/Cargo.lock') }} - - name: Download unsigned macOS DMG - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} + name: Install Zig + uses: mlugg/setup-zig@v2 with: - name: ${{ matrix.artifact_name }}-unsigned-dmg - path: ${{ runner.temp }}/unsigned-dmg + version: 0.14.0 - - name: Set up AKV PKCS11 macOS signing - uses: ./.github/actions/setup-akv-pkcs11-codesigning - with: - rcodesign-blob-uri: ${{ secrets.AKV_CODESIGN_RCODESIGN_BLOB_URI }} - rcodesign-sha256: ${{ secrets.AKV_CODESIGN_RCODESIGN_SHA256 }} - akv-pkcs11-library-blob-uri: ${{ secrets.AKV_CODESIGN_PKCS11_LIBRARY_BLOB_URI }} - akv-pkcs11-library-sha256: ${{ secrets.AKV_CODESIGN_PKCS11_LIBRARY_SHA256 }} - azure-client-id: ${{ secrets.AKV_CODESIGN_AZURE_CLIENT_ID }} - azure-tenant-id: ${{ secrets.AKV_CODESIGN_TENANT }} - azure-subscription-id: ${{ secrets.AKV_CODESIGN_SUBSCRIPTION }} - key-vault-name: ${{ secrets.AKV_CODESIGN_KEY_VAULT_NAME }} - key-name: ${{ secrets.AKV_CODESIGN_KEY_NAME }} - key-version: ${{ secrets.AKV_CODESIGN_KEY_VERSION || '' }} - certificate-sha256: ${{ secrets.AKV_CODESIGN_CERTIFICATE_SHA256 || '' }} - - - name: Sign, notarize, and staple macOS DMG - shell: bash + - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} + name: Install musl build tools env: TARGET: ${{ matrix.target }} - APPLE_NOTARIZATION_KEY_P8: ${{ secrets.APPLE_NOTARIZATION_KEY_P8 }} - APPLE_NOTARIZATION_KEY_ID: ${{ secrets.APPLE_NOTARIZATION_KEY_ID }} - APPLE_NOTARIZATION_ISSUER_ID: ${{ secrets.APPLE_NOTARIZATION_ISSUER_ID }} + run: bash "${GITHUB_WORKSPACE}/.github/scripts/install-musl-build-tools.sh" + + - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} + name: Configure rustc UBSan wrapper (musl host) + shell: bash run: | set -euo pipefail - - dmg_path="${RUNNER_TEMP}/unsigned-dmg/codex-${TARGET}.dmg" - report_dir="${GITHUB_WORKSPACE}/macos-dmg-signing-verification/${TARGET}" - if [[ ! -f "$dmg_path" ]]; then - echo "Unsigned DMG $dmg_path not found" - exit 1 + ubsan="" + if command -v ldconfig >/dev/null 2>&1; then + ubsan="$(ldconfig -p | grep -m1 'libubsan\.so\.1' | sed -E 's/.*=> (.*)$/\1/')" fi + wrapper_root="${RUNNER_TEMP:-/tmp}" + wrapper="${wrapper_root}/rustc-ubsan-wrapper" + cat > "${wrapper}" <> "$GITHUB_ENV" + echo "RUSTC_WORKSPACE_WRAPPER=" >> "$GITHUB_ENV" - .github/scripts/macos-signing/sign_macos_code.sh \ - --target "$dmg_path" \ - --identity unused \ - --deep false \ - --timestamp true - - mkdir -p "$report_dir" - rcodesign print-signature-info "$dmg_path" \ - >"${report_dir}/signature-info-before-notarization.yaml" - - .github/scripts/macos-signing/notarize_macos_dmg_with_rcodesign.sh \ - --dmg "$dmg_path" \ - --report-dir "$report_dir" - - rcodesign print-signature-info "$dmg_path" \ - >"${report_dir}/signature-info.yaml" + - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} + name: Clear sanitizer flags (musl) + shell: bash + run: | + set -euo pipefail + # Clear global Rust flags so host/proc-macro builds don't pull in UBSan. + echo "RUSTFLAGS=" >> "$GITHUB_ENV" + echo "CARGO_ENCODED_RUSTFLAGS=" >> "$GITHUB_ENV" + echo "RUSTDOCFLAGS=" >> "$GITHUB_ENV" + # Override any runner-level Cargo config rustflags as well. + echo "CARGO_BUILD_RUSTFLAGS=" >> "$GITHUB_ENV" + echo "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS=" >> "$GITHUB_ENV" + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS=" >> "$GITHUB_ENV" + echo "CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS=" >> "$GITHUB_ENV" + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS=" >> "$GITHUB_ENV" + + sanitize_flags() { + local input="$1" + input="${input//-fsanitize=undefined/}" + input="${input//-fno-sanitize-recover=undefined/}" + input="${input//-fno-sanitize-trap=undefined/}" + echo "$input" + } - - name: Upload signed macOS DMG - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: ${{ matrix.artifact_name }}-signed-dmg - path: ${{ runner.temp }}/unsigned-dmg/codex-${{ matrix.target }}.dmg - if-no-files-found: error + cflags="$(sanitize_flags "${CFLAGS-}")" + cxxflags="$(sanitize_flags "${CXXFLAGS-}")" + echo "CFLAGS=${cflags}" >> "$GITHUB_ENV" + echo "CXXFLAGS=${cxxflags}" >> "$GITHUB_ENV" - - name: Upload DMG signing verification - if: ${{ always() }} - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: ${{ matrix.artifact_name }}-dmg-signing-verification - path: macos-dmg-signing-verification/${{ matrix.target }}/ - if-no-files-found: warn + - name: Cargo build + shell: bash + run: | + set -euo pipefail - finalize-macos: - if: ${{ github.event_name != 'workflow_dispatch' }} - needs: - - package-macos - - sign-macos-dmg - name: Verify macOS artifacts - ${{ matrix.target }} - ${{ matrix.bundle }} - runs-on: macos-15-xlarge - timeout-minutes: 30 - permissions: - contents: read - defaults: - run: - working-directory: codex-rs + if [[ "${{ matrix.target }}" == 'aarch64-linux-android' ]]; then + export CARGO_BUILD_JOBS=4 + export CARGO_PROFILE_RELEASE_LTO=thin + export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=8 + cargo build --target ${{ matrix.target }} --release --bin codex + exit 0 + fi - strategy: - fail-fast: false - matrix: - include: - - target: aarch64-apple-darwin - bundle: primary - artifact_name: aarch64-apple-darwin - binaries: "codex codex-responses-api-proxy" - verify_dmg: "true" - - target: aarch64-apple-darwin - bundle: app-server - artifact_name: aarch64-apple-darwin-app-server - binaries: "codex-app-server" - verify_dmg: "false" - - target: x86_64-apple-darwin - bundle: primary - artifact_name: x86_64-apple-darwin - binaries: "codex codex-responses-api-proxy" - verify_dmg: "true" - - target: x86_64-apple-darwin - bundle: app-server - artifact_name: x86_64-apple-darwin-app-server - binaries: "codex-app-server" - verify_dmg: "false" + if [[ "${{ contains(matrix.target, 'windows') }}" == 'true' ]]; then + cargo build --target ${{ matrix.target }} --release --bin codex --bin codex-responses-api-proxy --bin codex-windows-sandbox-setup --bin codex-command-runner + else + cargo build --target ${{ matrix.target }} --release --bin codex --bin codex-responses-api-proxy + fi + - name: sccache stats (Android) + if: ${{ matrix.target == 'aarch64-linux-android' }} + shell: bash + run: ${SCCACHE_PATH:-sccache} --show-stats || true - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false + - if: ${{ matrix.target == 'aarch64-linux-android' }} + name: Normalize Android ELF for Termux + shell: bash + run: | + set -euo pipefail + for binary in codex codex-responses-api-proxy; do + binary_path="target/${{ matrix.target }}/release/${binary}" + if [[ -f "${binary_path}" ]]; then + termux-elf-cleaner --api-level 24 "${binary_path}" + chmod +x "${binary_path}" + # if [[ "${binary}" == "codex" ]]; then + # # Patch PT_TLS (0x7) to PT_NULL (0x0) at offset 400 to bypass Bionic alignment checks. + # printf '\x00\x00\x00\x00' | dd of="${binary_path}" bs=1 seek=400 count=4 conv=notrunc + # fi + fi + done - - name: Download packaged macOS artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - if: ${{ contains(matrix.target, 'linux') && !contains(matrix.target, 'android') && github.repository_owner == 'openai' }} + name: Cosign Linux artifacts + uses: ./.github/actions/linux-code-sign with: - name: ${{ matrix.artifact_name }}-packaged - path: codex-rs/dist/${{ matrix.target }} + target: ${{ matrix.target }} + artifacts-dir: ${{ github.workspace }}/codex-rs/target/${{ matrix.target }}/release - - name: Download signed macOS binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - if: ${{ contains(matrix.target, 'windows') && github.repository_owner == 'openai' }} + name: Sign Windows binaries with Azure Trusted Signing + uses: ./.github/actions/windows-code-sign with: - name: ${{ matrix.artifact_name }}-signed-binaries - path: ${{ runner.temp }}/signed-binaries - - - name: Download signed macOS DMG - if: ${{ matrix.verify_dmg == 'true' }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + target: ${{ matrix.target }} + client-id: ${{ secrets.AZURE_TRUSTED_SIGNING_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TRUSTED_SIGNING_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_TRUSTED_SIGNING_SUBSCRIPTION_ID }} + endpoint: ${{ secrets.AZURE_TRUSTED_SIGNING_ENDPOINT }} + account-name: ${{ secrets.AZURE_TRUSTED_SIGNING_ACCOUNT_NAME }} + certificate-profile-name: ${{ secrets.AZURE_TRUSTED_SIGNING_CERTIFICATE_PROFILE_NAME }} + + - if: ${{ runner.os == 'macOS' && github.repository_owner == 'openai' }} + name: MacOS code signing (binaries) + uses: ./.github/actions/macos-code-sign with: - name: ${{ matrix.artifact_name }}-signed-dmg - path: ${{ runner.temp }}/signed-dmg - - - name: Verify signed macOS artifacts + target: ${{ matrix.target }} + sign-binaries: "true" + sign-dmg: "false" + apple-certificate: ${{ secrets.APPLE_CERTIFICATE_P12 }} + apple-certificate-password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} + apple-notarization-key-p8: ${{ secrets.APPLE_NOTARIZATION_KEY_P8 }} + apple-notarization-key-id: ${{ secrets.APPLE_NOTARIZATION_KEY_ID }} + apple-notarization-issuer-id: ${{ secrets.APPLE_NOTARIZATION_ISSUER_ID }} + + - if: ${{ runner.os == 'macOS' && github.repository_owner == 'openai' }} + name: Build macOS dmg shell: bash run: | set -euo pipefail target="${{ matrix.target }}" - packaged_dir="dist/${target}" - expected_entitlements="${GITHUB_WORKSPACE}/.github/scripts/macos-signing/codex.entitlements.plist" - - verify_signed_binary() { - local path="$1" - local actual_entitlements normalized_actual normalized_expected - - chmod 0755 "$path" - codesign --verify --strict --verbose=2 "$path" - - actual_entitlements="$(mktemp)" - normalized_actual="$(mktemp)" - normalized_expected="$(mktemp)" - codesign -d --entitlements :- "$path" >"$actual_entitlements" - plutil -convert xml1 -o "$normalized_actual" "$actual_entitlements" - plutil -convert xml1 -o "$normalized_expected" "$expected_entitlements" - diff -u "$normalized_expected" "$normalized_actual" - rm -f "$actual_entitlements" "$normalized_actual" "$normalized_expected" - } - - for binary in ${{ matrix.binaries }}; do - binary_path="${RUNNER_TEMP}/signed-binaries/${binary}" - verify_signed_binary "$binary_path" - - direct_archive_dir="${RUNNER_TEMP}/direct-archive-${binary}-${target}" - rm -rf "$direct_archive_dir" - mkdir -p "$direct_archive_dir" - tar -xzf "${packaged_dir}/${binary}-${target}.tar.gz" -C "$direct_archive_dir" - verify_signed_binary "${direct_archive_dir}/${binary}-${target}" + release_dir="target/${target}/release" + dmg_root="${RUNNER_TEMP}/codex-dmg-root" + volname="Codex (${target})" + dmg_path="${release_dir}/codex-${target}.dmg" - direct_zstd_path="${RUNNER_TEMP}/${binary}-${target}-from-zstd" - zstd -d --stdout "${packaged_dir}/${binary}-${target}.zst" >"$direct_zstd_path" - verify_signed_binary "$direct_zstd_path" - done + # The previous "MacOS code signing (binaries)" step signs + notarizes the + # built artifacts in `${release_dir}`. This step packages *those same* + # signed binaries into a dmg. + codex_binary_path="${release_dir}/codex" + proxy_binary_path="${release_dir}/codex-responses-api-proxy" - case "${{ matrix.bundle }}" in - primary) - package_stem="codex-package" - package_entrypoint="codex" - ;; - app-server) - package_stem="codex-app-server-package" - package_entrypoint="codex-app-server" - ;; - *) - echo "Unexpected macOS bundle: ${{ matrix.bundle }}" - exit 1 - ;; - esac - - package_dir="${RUNNER_TEMP}/${package_stem}-${target}" - rm -rf "$package_dir" - mkdir -p "$package_dir" - tar -xzf "${packaged_dir}/${package_stem}-${target}.tar.gz" -C "$package_dir" - verify_signed_binary "${package_dir}/bin/${package_entrypoint}" + rm -rf "$dmg_root" + mkdir -p "$dmg_root" - if [[ "${{ matrix.verify_dmg }}" != "true" ]]; then - exit 0 + if [[ ! -f "$codex_binary_path" ]]; then + echo "Binary $codex_binary_path not found" + exit 1 fi - - dmg_path="${RUNNER_TEMP}/signed-dmg/codex-${target}.dmg" - mount_dir="${RUNNER_TEMP}/codex-dmg-mount-${target}" - if [[ ! -f "$dmg_path" ]]; then - echo "Signed DMG $dmg_path not found" + if [[ ! -f "$proxy_binary_path" ]]; then + echo "Binary $proxy_binary_path not found" exit 1 fi - hdiutil verify "$dmg_path" - codesign --verify --strict --verbose=2 "$dmg_path" - xcrun stapler validate "$dmg_path" - - rm -rf "$mount_dir" - mkdir -p "$mount_dir" - hdiutil attach "$dmg_path" -nobrowse -readonly -mountpoint "$mount_dir" - cleanup_mount() { - hdiutil detach "$mount_dir" >/dev/null - } - trap cleanup_mount EXIT - - for binary in ${{ matrix.binaries }}; do - verify_signed_binary "${mount_dir}/${binary}" - done - - cleanup_mount - trap - EXIT - cp "$dmg_path" "dist/${target}/codex-${target}.dmg" + ditto "$codex_binary_path" "${dmg_root}/codex" + ditto "$proxy_binary_path" "${dmg_root}/codex-responses-api-proxy" - - name: Upload verified macOS artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: ${{ matrix.artifact_name }} - path: codex-rs/dist/${{ matrix.target }}/* - if-no-files-found: error - - stage-signed-macos: - if: ${{ github.event_name == 'workflow_dispatch' && inputs.release_mode == 'promote_signed' }} - needs: tag-check - name: Stage signed macOS handoff - ${{ matrix.target }} - ${{ matrix.bundle }} - runs-on: macos-15-xlarge - timeout-minutes: 30 - permissions: - contents: read - defaults: - run: - working-directory: codex-rs + rm -f "$dmg_path" + hdiutil create \ + -volname "$volname" \ + -srcfolder "$dmg_root" \ + -format UDZO \ + -ov \ + "$dmg_path" - strategy: - fail-fast: false - matrix: - include: - - target: aarch64-apple-darwin - bundle: primary - artifact_name: aarch64-apple-darwin - binaries: "codex codex-responses-api-proxy" - build_dmg: "false" - - target: aarch64-apple-darwin - bundle: app-server - artifact_name: aarch64-apple-darwin-app-server - binaries: "codex-app-server" - build_dmg: "false" - - target: x86_64-apple-darwin - bundle: primary - artifact_name: x86_64-apple-darwin - binaries: "codex codex-responses-api-proxy" - build_dmg: "false" - - target: x86_64-apple-darwin - bundle: app-server - artifact_name: x86_64-apple-darwin-app-server - binaries: "codex-app-server" - build_dmg: "false" + if [[ ! -f "$dmg_path" ]]; then + echo "dmg $dmg_path not found after build" + exit 1 + fi - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - if: ${{ runner.os == 'macOS' && github.repository_owner == 'openai' }} + name: MacOS code signing (dmg) + uses: ./.github/actions/macos-code-sign with: - persist-credentials: false + target: ${{ matrix.target }} + sign-binaries: "false" + sign-dmg: "true" + apple-certificate: ${{ secrets.APPLE_CERTIFICATE_P12 }} + apple-certificate-password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} + apple-notarization-key-p8: ${{ secrets.APPLE_NOTARIZATION_KEY_P8 }} + apple-notarization-key-id: ${{ secrets.APPLE_NOTARIZATION_KEY_ID }} + apple-notarization-issuer-id: ${{ secrets.APPLE_NOTARIZATION_ISSUER_ID }} - - name: Download signed macOS handoff + - name: Stage artifacts shell: bash - env: - GH_TOKEN: ${{ github.token }} - SIGNED_MACOS_ASSET: ${{ inputs.signed_macos_asset }} - SIGNED_MACOS_SHA256: ${{ inputs.signed_macos_sha256 }} run: | - set -euo pipefail - - download_dir="${RUNNER_TEMP}/signed-macos-download" - handoff_dir="${RUNNER_TEMP}/signed-macos-handoff" - rm -rf "$download_dir" "$handoff_dir" - mkdir -p "$download_dir" "$handoff_dir" - - gh release download "$GITHUB_REF_NAME" \ - --repo "$GITHUB_REPOSITORY" \ - --pattern "$SIGNED_MACOS_ASSET" \ - --dir "$download_dir" + dest="dist/${{ matrix.target }}" + mkdir -p "$dest" - asset_count="$(find "$download_dir" -maxdepth 1 -type f | wc -l | tr -d '[:space:]')" - if [[ "$asset_count" != "1" ]]; then - echo "Expected exactly one signed macOS handoff asset named ${SIGNED_MACOS_ASSET}; found ${asset_count}" - find "$download_dir" -maxdepth 1 -type f -print - exit 1 + if [[ "${{ matrix.runner }}" == windows* ]]; then + cp target/${{ matrix.target }}/release/codex.exe "$dest/codex-${{ matrix.target }}.exe" + cp target/${{ matrix.target }}/release/codex-responses-api-proxy.exe "$dest/codex-responses-api-proxy-${{ matrix.target }}.exe" + cp target/${{ matrix.target }}/release/codex-windows-sandbox-setup.exe "$dest/codex-windows-sandbox-setup-${{ matrix.target }}.exe" + cp target/${{ matrix.target }}/release/codex-command-runner.exe "$dest/codex-command-runner-${{ matrix.target }}.exe" + else + # For Android, we want the binary to be named just 'codex' in the archive. + if [[ "${{ matrix.target }}" == "aarch64-linux-android" ]]; then + cp target/${{ matrix.target }}/release/codex "$dest/codex" + else + cp target/${{ matrix.target }}/release/codex "$dest/codex-${{ matrix.target }}" + fi + if [[ -f target/${{ matrix.target }}/release/codex-responses-api-proxy ]]; then + cp target/${{ matrix.target }}/release/codex-responses-api-proxy "$dest/codex-responses-api-proxy-${{ matrix.target }}" + fi fi - asset_path="$(find "$download_dir" -maxdepth 1 -type f -print -quit)" - if [[ -n "${SIGNED_MACOS_SHA256}" ]]; then - expected_sha="$(printf '%s' "$SIGNED_MACOS_SHA256" | tr '[:upper:]' '[:lower:]')" - actual_sha="$(shasum -a 256 "$asset_path" | awk '{print $1}')" - if [[ "$actual_sha" != "$expected_sha" ]]; then - echo "signed_macos_sha256 mismatch for ${SIGNED_MACOS_ASSET}" - echo "expected: ${expected_sha}" - echo "actual: ${actual_sha}" - exit 1 - fi + if [[ "${{ matrix.target }}" == *linux* && "${{ matrix.target }}" != *android* ]]; then + cp target/${{ matrix.target }}/release/codex.sigstore "$dest/codex-${{ matrix.target }}.sigstore" + cp target/${{ matrix.target }}/release/codex-responses-api-proxy.sigstore "$dest/codex-responses-api-proxy-${{ matrix.target }}.sigstore" fi - asset_name="$(basename "$asset_path")" - case "$asset_name" in - *.tar.zst) - zstd -dc "$asset_path" | tar -C "$handoff_dir" -xf - - ;; - *.tar.gz|*.tgz) - tar -C "$handoff_dir" -xzf "$asset_path" - ;; - *.zip) - ditto -x -k "$asset_path" "$handoff_dir" - ;; - *) - echo "Unsupported signed macOS handoff archive format: ${asset_name}" - exit 1 - ;; - esac + if [[ "${{ matrix.target }}" == *apple-darwin ]]; then + cp target/${{ matrix.target }}/release/codex-${{ matrix.target }}.dmg "$dest/codex-${{ matrix.target }}.dmg" + fi - echo "SIGNED_MACOS_HANDOFF_DIR=$handoff_dir" >> "$GITHUB_ENV" + - if: ${{ matrix.runner == 'windows-11-arm' }} + name: Install zstd + shell: powershell + run: choco install -y zstandard - - name: Stage signed macOS artifacts + - name: Compress artifacts shell: bash run: | - set -euo pipefail - - target="${{ matrix.target }}" - artifact_name="${{ matrix.artifact_name }}" - source_dir="${SIGNED_MACOS_HANDOFF_DIR}/${artifact_name}" - if [[ ! -d "$source_dir" && -d "${SIGNED_MACOS_HANDOFF_DIR}/dist/${artifact_name}" ]]; then - source_dir="${SIGNED_MACOS_HANDOFF_DIR}/dist/${artifact_name}" - fi - if [[ ! -d "$source_dir" && -d "${SIGNED_MACOS_HANDOFF_DIR}/${target}" ]]; then - source_dir="${SIGNED_MACOS_HANDOFF_DIR}/${target}" - fi - if [[ ! -d "$source_dir" && -d "${SIGNED_MACOS_HANDOFF_DIR}/dist/${target}" ]]; then - source_dir="${SIGNED_MACOS_HANDOFF_DIR}/dist/${target}" - fi - if [[ ! -d "$source_dir" ]]; then - echo "Signed macOS handoff is missing ${artifact_name}/" - echo "Expected either:" - echo " ${SIGNED_MACOS_HANDOFF_DIR}/${artifact_name}" - echo " ${SIGNED_MACOS_HANDOFF_DIR}/dist/${artifact_name}" - echo " ${SIGNED_MACOS_HANDOFF_DIR}/${target}" - echo " ${SIGNED_MACOS_HANDOFF_DIR}/dist/${target}" - find "$SIGNED_MACOS_HANDOFF_DIR" -maxdepth 3 -type f -print - exit 1 + # Path that contains the uncompressed binaries for the current + # ${{ matrix.target }} + dest="dist/${{ matrix.target }}" + repo_root=$PWD + + # We want to ship the raw Windows executables in the GitHub Release + # in addition to the compressed archives. Keep the originals for + # Windows targets; remove them elsewhere to limit the number of + # artifacts that end up in the GitHub Release. + keep_originals=false + if [[ "${{ matrix.runner }}" == windows* ]]; then + keep_originals=true fi - dest="dist/${target}" - mkdir -p "$dest" + # For compatibility with environments that lack the `zstd` tool we + # additionally create a `.tar.gz` for all platforms and `.zip` for + # Windows alongside every single binary that we publish. The end result is: + # codex-.zst (existing) + # codex-.tar.gz (new) + # codex-.zip (only for Windows) - for binary in ${{ matrix.binaries }}; do - source_path="${source_dir}/${binary}" - if [[ ! -f "$source_path" ]]; then - source_path="${source_dir}/${binary}-${target}" + # 1. Produce a .tar.gz for every file in the directory *before* we + # run `zstd --rm`, because that flag deletes the original files. + for f in "$dest"/*; do + base="$(basename "$f")" + # Skip files that are already archives (shouldn't happen, but be + # safe). + if [[ "$base" == *.tar.gz || "$base" == *.zip || "$base" == *.dmg ]]; then + continue fi - if [[ ! -f "$source_path" ]]; then - echo "Signed macOS handoff is missing ${binary} for ${artifact_name}" - exit 1 + + # Don't try to compress signature bundles. + if [[ "$base" == *.sigstore ]]; then + continue fi - release_path="${dest}/${binary}-${target}" - ditto "$source_path" "$release_path" - chmod 0755 "$release_path" - codesign --verify --strict --verbose=2 "$release_path" - done + # Create per-binary tar.gz + # For Android, we want the archive to have the target suffix even though the binary is just 'codex'. + archive_name="${base}" + if [[ "${{ matrix.target }}" == "aarch64-linux-android" && "${base}" == "codex" ]]; then + archive_name="codex-${{ matrix.target }}" + fi + tar -C "$dest" -czf "$dest/${archive_name}.tar.gz" "$base" + + # Create zip archive for Windows binaries + # Must run from inside the dest dir so 7z won't + # embed the directory path inside the zip. + if [[ "${{ matrix.runner }}" == windows* ]]; then + if [[ "$base" == "codex-${{ matrix.target }}.exe" ]]; then + # Bundle the sandbox helper binaries into the main codex zip so + # WinGet installs include the required helpers next to codex.exe. + # Fall back to the single-binary zip if the helpers are missing + # to avoid breaking releases. + bundle_dir="$(mktemp -d)" + runner_src="$dest/codex-command-runner-${{ matrix.target }}.exe" + setup_src="$dest/codex-windows-sandbox-setup-${{ matrix.target }}.exe" + if [[ -f "$runner_src" && -f "$setup_src" ]]; then + cp "$dest/$base" "$bundle_dir/$base" + cp "$runner_src" "$bundle_dir/codex-command-runner.exe" + cp "$setup_src" "$bundle_dir/codex-windows-sandbox-setup.exe" + # Use an absolute path so bundle zips land in the real dist + # dir even when 7z runs from a temp directory. + (cd "$bundle_dir" && 7z a "$repo_root/$dest/${base}.zip" .) + else + echo "warning: missing sandbox binaries; falling back to single-binary zip" + echo "warning: expected $runner_src and $setup_src" + (cd "$dest" && 7z a "${base}.zip" "$base") + fi + rm -rf "$bundle_dir" + else + (cd "$dest" && 7z a "${base}.zip" "$base") + fi + fi - # DMG staging is disabled for signed promotion because we no longer - # distribute DMGs from this release path. Keep the branch here so the - # handoff can opt back in by flipping matrix.build_dmg if needed. - if [[ "${{ matrix.build_dmg }}" == "true" ]]; then - dmg_name="codex-${target}.dmg" - dmg_source="${source_dir}/${dmg_name}" - if [[ ! -f "$dmg_source" ]]; then - echo "Signed macOS handoff is missing ${dmg_name} for ${artifact_name}" - exit 1 + # Also create .zst (existing behaviour) *and* remove the original + # uncompressed binary to keep the directory small. + zstd_args=(-T0 -19) + if [[ "${keep_originals}" == false ]]; then + zstd_args+=(--rm) fi - codesign --verify --strict --verbose=2 "$dmg_source" - xcrun stapler validate "$dmg_source" - cp "$dmg_source" "$dest/$dmg_name" - fi + if [[ "${archive_name}" != "${base}" ]]; then + zstd "${zstd_args[@]}" "$dest/$base" -o "$dest/${archive_name}.zst" + else + zstd "${zstd_args[@]}" "$dest/$base" + fi + done - - name: Build Codex package archive - shell: bash - env: - TARGET: ${{ matrix.target }} - BUNDLE: ${{ matrix.bundle }} - run: | - set -euo pipefail - bash "${GITHUB_WORKSPACE}/.github/scripts/build-codex-package-archive.sh" \ - --target "$TARGET" \ - --bundle "$BUNDLE" \ - --entrypoint-dir "dist/${TARGET}" \ - --archive-dir "dist/${TARGET}" \ - --target-suffixed-entrypoint - - - name: Build Python runtime wheel - if: ${{ matrix.bundle == 'primary' }} + - name: Add Termux release metadata + if: ${{ (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') && matrix.target == 'aarch64-linux-android' }} shell: bash run: | set -euo pipefail + if [[ ! -f "${GITHUB_WORKSPACE}/.github/termux-release.json" ]]; then + echo "No Termux release metadata found; skipping metadata attachment." + exit 0 + fi + dest="dist/${{ matrix.target }}" + cp "${GITHUB_WORKSPACE}/.github/termux-release.json" "${dest}/termux-release.json" + ( + cd "${dest}" + sha256sum ./* > SHA256SUMS + ) - case "${{ matrix.target }}" in - aarch64-apple-darwin) - platform_tag="macosx_11_0_arm64" - ;; - x86_64-apple-darwin) - platform_tag="macosx_10_9_x86_64" - ;; - *) - echo "No Python runtime wheel platform tag for ${{ matrix.target }}" - exit 1 - ;; - esac - - python3 -m venv "${RUNNER_TEMP}/python-runtime-build-venv" - "${RUNNER_TEMP}/python-runtime-build-venv/bin/python" -m pip install build - - stage_dir="${RUNNER_TEMP}/openai-codex-cli-bin-${{ matrix.target }}" - wheel_dir="${GITHUB_WORKSPACE}/python-runtime-dist/${{ matrix.target }}" - python3 \ - "${GITHUB_WORKSPACE}/sdk/python/scripts/update_sdk_artifacts.py" \ - stage-runtime \ - "$stage_dir" \ - "dist/${{ matrix.target }}/codex-package-${{ matrix.target }}.tar.gz" \ - --codex-version "${GITHUB_REF_NAME}" \ - --platform-tag "$platform_tag" - "${RUNNER_TEMP}/python-runtime-build-venv/bin/python" -m build --wheel --outdir "$wheel_dir" "$stage_dir" - - - name: Upload Python runtime wheel - if: ${{ matrix.bundle == 'primary' }} - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + - id: upload-artifact + uses: actions/upload-artifact@v6 with: - name: python-runtime-wheel-${{ matrix.target }} - path: python-runtime-dist/${{ matrix.target }}/*.whl - if-no-files-found: error + name: ${{ github.event_name == 'pull_request' && format('termux-android-pr-{0}-{1}', github.event.pull_request.number, github.event.pull_request.head.sha) || matrix.target }} + # Upload the per-binary .zst files as well as the new .tar.gz + # equivalents we generated in the previous step. + path: | + codex-rs/dist/${{ matrix.target }}/* - - name: Compress artifacts + - name: Publish Termux artifact notification + if: ${{ github.event_name == 'pull_request' && startsWith(github.base_ref, 'release/') && matrix.target == 'aarch64-linux-android' }} shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ARTIFACT_ID: ${{ steps.upload-artifact.outputs.artifact-id }} + ARTIFACT_NAME: ${{ github.event_name == 'pull_request' && format('termux-android-pr-{0}-{1}', github.event.pull_request.number, github.event.pull_request.head.sha) || matrix.target }} + PR_NUMBER: ${{ github.event.pull_request.number }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} run: | set -euo pipefail + if [[ -z "${ARTIFACT_ID}" ]]; then + echo "upload-artifact did not return an artifact id" >&2 + exit 1 + fi - dest="dist/${{ matrix.target }}" - for f in "$dest"/*; do - base="$(basename "$f")" - if [[ "$base" == *.tar.gz || "$base" == *.tar.zst || "$base" == *.zip || "$base" == *.dmg ]]; then - continue - fi - - tar -C "$dest" -czf "$dest/${base}.tar.gz" "$base" - zstd -T0 -19 --rm "$dest/$base" - done - - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: ${{ matrix.artifact_name }} - path: | - codex-rs/dist/${{ matrix.target }}/* + marker="" + artifact_url="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts/${ARTIFACT_ID}" + run_url="${GH_WORKFLOW_URL:-https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}}" + body_path="${RUNNER_TEMP}/termux-artifact-comment.md" + { + echo "${marker}" + echo "Termux Android artifact ready for testing:" + echo + echo "- Artifact: [${ARTIFACT_NAME}](${artifact_url})" + echo "- Workflow run: [${GITHUB_RUN_ID}](${run_url})" + echo + echo "You must be signed in to GitHub with repository access to download Actions artifacts." + } > "${body_path}" + + existing_comment_id="$( + gh pr view "${PR_NUMBER}" \ + --repo "${GITHUB_REPOSITORY}" \ + --json comments \ + --jq ".comments | map(select(.author.is_bot == true and (.body | contains(\"${marker}\")))) | .[-1].id // \"\"" + )" + if [[ -n "${existing_comment_id}" ]]; then + gh api graphql \ + -f query=' + mutation($id: ID!, $body: String!) { + updateIssueComment(input: {id: $id, body: $body}) { + issueComment { + id + } + } + } + ' \ + -f id="${existing_comment_id}" \ + -f body="$(cat "${body_path}")" \ + >/dev/null + else + gh pr comment "${PR_NUMBER}" --repo "${GITHUB_REPOSITORY}" --body-file "${body_path}" + fi - build-windows: - if: ${{ github.event_name != 'workflow_dispatch' || inputs.release_mode == 'build_unsigned' }} - needs: tag-check - uses: ./.github/workflows/rust-release-windows.yml - with: - release-lto: ${{ contains(github.ref_name, '-alpha') && 'thin' || 'fat' }} - secrets: inherit + gh label create binary-ready \ + --repo "${GITHUB_REPOSITORY}" \ + --color 0e8a16 \ + --description "Android binary is ready for testing" \ + --force + gh pr edit "${PR_NUMBER}" --repo "${GITHUB_REPOSITORY}" --add-label binary-ready + + if [[ -n "${SLACK_WEBHOOK_URL:-}" ]]; then + slack_payload_path="${RUNNER_TEMP}/termux-artifact-slack.json" + pr_url="https://github.com/${GITHUB_REPOSITORY}/pull/${PR_NUMBER}" + jq -n \ + --arg artifact_name "${ARTIFACT_NAME}" \ + --arg artifact_url "${artifact_url}" \ + --arg pr_number "${PR_NUMBER}" \ + --arg pr_url "${pr_url}" \ + --arg run_id "${GITHUB_RUN_ID}" \ + --arg run_url "${run_url}" \ + '{ + text: ("Termux Android artifact ready for testing: " + $artifact_url), + blocks: [ + { + type: "section", + text: { + type: "mrkdwn", + text: ("*Termux Android artifact ready for testing*\n" + $artifact_url) + } + }, + { + type: "section", + fields: [ + { + type: "mrkdwn", + text: ("*Artifact:*\n<" + $artifact_url + "|" + $artifact_name + ">") + }, + { + type: "mrkdwn", + text: ("*Pull request:*\n<" + $pr_url + "|#" + $pr_number + ">") + }, + { + type: "mrkdwn", + text: ("*Workflow run:*\n<" + $run_url + "|" + $run_id + ">") + } + ] + }, + { + type: "context", + elements: [ + { + type: "mrkdwn", + text: "You must be signed in to GitHub with repository access to download Actions artifacts." + } + ] + } + ] + }' > "${slack_payload_path}" + + if ! curl -fsS \ + -X POST \ + -H "Content-type: application/json" \ + --data @"${slack_payload_path}" \ + "${SLACK_WEBHOOK_URL}"; then + echo "::warning::Failed to send Slack artifact notification" + fi + fi - argument-comment-lint-release-assets: - if: ${{ github.event_name != 'workflow_dispatch' || inputs.release_mode == 'build_unsigned' }} - name: argument-comment-lint release assets + shell-tool-mcp: + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && github.repository_owner == 'openai' + name: shell-tool-mcp needs: tag-check - uses: ./.github/workflows/rust-release-argument-comment-lint.yml + permissions: + contents: read + id-token: write + uses: ./.github/workflows/shell-tool-mcp.yml with: + release-tag: ${{ github.ref_name }} publish: true - - zsh-release-assets: - if: ${{ github.event_name != 'workflow_dispatch' || inputs.release_mode == 'build_unsigned' }} - name: zsh release assets - needs: tag-check - uses: ./.github/workflows/rust-release-zsh.yml + secrets: inherit release: + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') needs: - - tag-check - build - - finalize-macos - - stage-signed-macos - - build-windows - - argument-comment-lint-release-assets - - zsh-release-assets - if: >- - ${{ - always() && - needs.tag-check.result == 'success' && - ( - ( - github.event_name == 'workflow_dispatch' && - inputs.release_mode == 'promote_signed' && - needs.stage-signed-macos.result == 'success' && - needs.build.result == 'skipped' && - needs.finalize-macos.result == 'skipped' && - needs.build-windows.result == 'skipped' && - needs.argument-comment-lint-release-assets.result == 'skipped' && - needs.zsh-release-assets.result == 'skipped' - ) || - ( - (github.event_name != 'workflow_dispatch' || inputs.release_mode != 'promote_signed') && - needs.build.result == 'success' && - ( - ( - github.event_name == 'workflow_dispatch' && - inputs.release_mode == 'build_unsigned' && - needs.finalize-macos.result == 'skipped' - ) || - ( - github.event_name != 'workflow_dispatch' && - needs.finalize-macos.result == 'success' - ) - ) && - needs.stage-signed-macos.result == 'skipped' && - needs.build-windows.result == 'success' && - needs.argument-comment-lint-release-assets.result == 'success' && - needs.zsh-release-assets.result == 'success' - ) - ) - }} + - shell-tool-mcp name: release runs-on: ubuntu-latest permissions: contents: write actions: read - env: - RELEASE_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.release_mode || 'signed' }} - SIGN_MACOS: ${{ github.event_name != 'workflow_dispatch' || inputs.release_mode == 'promote_signed' }} - SIGNED_MACOS_ASSET: ${{ inputs.signed_macos_asset }} - UNSIGNED_RUN_ID: ${{ inputs.unsigned_run_id }} outputs: version: ${{ steps.release_name.outputs.name }} tag: ${{ github.ref_name }} - sign_macos: ${{ steps.release_mode.outputs.sign_macos }} should_publish_npm: ${{ steps.npm_publish_settings.outputs.should_publish }} npm_tag: ${{ steps.npm_publish_settings.outputs.npm_tag }} steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false + uses: actions/checkout@v6 - - name: Define release mode - id: release_mode - run: | - echo "release_mode=${RELEASE_MODE}" >> "$GITHUB_OUTPUT" - echo "sign_macos=${SIGN_MACOS}" >> "$GITHUB_OUTPUT" + - name: 🧰 Actions Toolbox + # This is required for the GitHub CLI + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: wallentx/gh-actions/composite/actions-toolbox@main - name: Generate release notes from tag commit message id: release_notes @@ -1442,168 +1047,21 @@ jobs: echo "path=${notes_path}" >> "${GITHUB_OUTPUT}" - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - uses: actions/download-artifact@v7 with: path: dist - - name: Validate unsigned build run - if: ${{ env.RELEASE_MODE == 'promote_signed' }} - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - - run_summary="$(gh run view "$UNSIGNED_RUN_ID" \ - --repo "$GITHUB_REPOSITORY" \ - --json conclusion,event,headBranch,headSha,status,workflowName,url \ - --jq '[.workflowName, .event, .headBranch, .headSha, .status, .conclusion, .url] | @tsv')" - IFS=$'\t' read -r workflow_name event head_branch head_sha status conclusion run_url <<< "$run_summary" - expected_head_sha="$(git rev-parse "${GITHUB_SHA}^{commit}")" - - if [[ "$workflow_name" != "$GITHUB_WORKFLOW" ]]; then - echo "unsigned_run_id ${UNSIGNED_RUN_ID} is for workflow '${workflow_name}', expected '${GITHUB_WORKFLOW}'" - echo "Run URL: ${run_url}" - exit 1 - fi - - if [[ "$event" != "workflow_dispatch" ]]; then - echo "unsigned_run_id ${UNSIGNED_RUN_ID} was triggered by '${event}', expected 'workflow_dispatch'" - echo "Run URL: ${run_url}" - exit 1 - fi - - if [[ "$head_branch" != "$GITHUB_REF_NAME" ]]; then - echo "unsigned_run_id ${UNSIGNED_RUN_ID} used ref '${head_branch}', expected '${GITHUB_REF_NAME}'" - echo "Run URL: ${run_url}" - exit 1 - fi - - if [[ "$head_sha" != "$expected_head_sha" ]]; then - echo "unsigned_run_id ${UNSIGNED_RUN_ID} used head SHA '${head_sha}', expected '${expected_head_sha}'" - echo "Run URL: ${run_url}" - exit 1 - fi - - if [[ "$status" != "completed" || "$conclusion" != "success" ]]; then - echo "unsigned_run_id ${UNSIGNED_RUN_ID} is ${status}/${conclusion}, expected completed/success" - echo "Run URL: ${run_url}" - exit 1 - fi - - - name: Download artifacts from unsigned build run - if: ${{ env.RELEASE_MODE == 'promote_signed' }} - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - gh run download "$UNSIGNED_RUN_ID" \ - --repo "$GITHUB_REPOSITORY" \ - --dir dist - - - name: Remove unsigned macOS staging artifacts - if: ${{ env.RELEASE_MODE == 'promote_signed' }} - run: | - set -euo pipefail - find dist -mindepth 1 -maxdepth 1 -type d \ - -name '*-apple-darwin*-unsigned' \ - -exec rm -rf {} + - - - name: Re-upload promoted Linux x64 artifacts - if: ${{ env.RELEASE_MODE == 'promote_signed' }} - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: x86_64-unknown-linux-musl - path: dist/x86_64-unknown-linux-musl/* - if-no-files-found: error - - - name: Re-upload promoted Linux arm64 artifacts - if: ${{ env.RELEASE_MODE == 'promote_signed' }} - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: aarch64-unknown-linux-musl - path: dist/aarch64-unknown-linux-musl/* - if-no-files-found: error - - - name: Re-upload promoted Windows x64 artifacts - if: ${{ env.RELEASE_MODE == 'promote_signed' }} - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: x86_64-pc-windows-msvc - path: dist/x86_64-pc-windows-msvc/* - if-no-files-found: error - - - name: Re-upload promoted Windows arm64 artifacts - if: ${{ env.RELEASE_MODE == 'promote_signed' }} - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: aarch64-pc-windows-msvc - path: dist/aarch64-pc-windows-msvc/* - if-no-files-found: error - - name: List run: ls -R dist/ - - name: Prune artifacts excluded from unsigned macOS release - if: ${{ env.SIGN_MACOS == 'false' }} - run: | - find dist -mindepth 1 -maxdepth 1 -type d \ - ! -name '*-apple-darwin*-unsigned' \ - ! -name 'aarch64-unknown-linux-musl' \ - ! -name 'aarch64-unknown-linux-musl-app-server' \ - ! -name 'x86_64-unknown-linux-musl' \ - ! -name 'x86_64-unknown-linux-musl-app-server' \ - ! -name 'aarch64-pc-windows-msvc' \ - ! -name 'x86_64-pc-windows-msvc' \ - -exec rm -rf {} + - - if ! find dist -type f -name '*-apple-darwin*-unsigned*' | grep -q .; then - echo "No unsigned macOS artifacts found in downloaded workflow artifacts." - exit 1 - fi - + # This is a temporary fix: we should modify shell-tool-mcp.yml so these + # files do not end up in dist/ in the first place. - name: Delete entries from dist/ that should not go in the release run: | - rm -rf dist/windows-binaries* - rm -rf dist/*-apple-darwin*-signed-binaries - rm -rf dist/*-apple-darwin*-packaged - rm -rf dist/*-apple-darwin*-unsigned-dmg - rm -rf dist/*-apple-darwin*-signed-dmg - rm -rf dist/*-apple-darwin*-binary-signing-verification - rm -rf dist/*-apple-darwin*-dmg-signing-verification - if [[ "${SIGN_MACOS}" == "true" ]]; then - rm -rf dist/*-apple-darwin*-unsigned - fi - # cargo-timing.html appears under multiple target-specific directories. - # If included in files: dist/**, release upload races on duplicate - # asset names and can fail with 404s. - find dist -type f -name 'cargo-timing.html' -delete - find dist -type d -empty -delete + rm -rf dist/shell-tool-mcp* ls -R dist/ - - name: Add Codex package checksum manifest - run: | - set -euo pipefail - - manifest="dist/codex-package_SHA256SUMS" - tmp_manifest="$(mktemp)" - find dist -type f \ - \( -name 'codex-package-*.tar.gz' -o -name 'codex-app-server-package-*.tar.gz' \) \ - -print | - sort | - while IFS= read -r archive; do - sha256sum "$archive" | - awk -v name="$(basename "$archive")" '{ print $1 " " name }' - done > "$tmp_manifest" - - if [[ ! -s "$tmp_manifest" ]]; then - echo "No Codex package archives found for checksum manifest" - exit 1 - fi - - mv "$tmp_manifest" "$manifest" - cat "$manifest" - - name: Add config schema release asset run: | cp codex-rs/core/config.schema.json dist/config-schema.json @@ -1624,12 +1082,6 @@ jobs: set -euo pipefail version="${VERSION}" - if [[ "${SIGN_MACOS}" != "true" ]]; then - echo "should_publish=false" >> "$GITHUB_OUTPUT" - echo "npm_tag=" >> "$GITHUB_OUTPUT" - exit 0 - fi - if [[ "${version}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then echo "should_publish=true" >> "$GITHUB_OUTPUT" echo "npm_tag=" >> "$GITHUB_OUTPUT" @@ -1642,118 +1094,69 @@ jobs: fi - name: Setup pnpm - if: ${{ env.SIGN_MACOS == 'true' }} - uses: pnpm/action-setup@a8198c4bff370c8506180b035930dea56dbd5288 # v5 + uses: pnpm/action-setup@v4 with: run_install: false - name: Setup Node.js for npm packaging - if: ${{ env.SIGN_MACOS == 'true' }} - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + uses: actions/setup-node@v6 with: node-version: 22 - name: Install dependencies - if: ${{ env.SIGN_MACOS == 'true' }} run: pnpm install --frozen-lockfile + # stage_npm_packages.py requires DotSlash when staging releases. + - uses: facebook/install-dotslash@v2 - name: Stage npm packages - if: ${{ env.SIGN_MACOS == 'true' }} + if: github.repository_owner == 'openai' env: GH_TOKEN: ${{ github.token }} - RELEASE_VERSION: ${{ steps.release_name.outputs.name }} run: | - workflow_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" ./scripts/stage_npm_packages.py \ - --release-version "$RELEASE_VERSION" \ - --workflow-url "$workflow_url" \ + --release-version "${{ steps.release_name.outputs.name }}" \ --package codex \ --package codex-responses-api-proxy \ --package codex-sdk - - name: Stage installer scripts - if: ${{ env.SIGN_MACOS == 'true' }} - run: | - cp scripts/install/install.sh dist/install.sh - cp scripts/install/install.ps1 dist/install.ps1 - - name: Create GitHub Release - uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2.6.1 + uses: softprops/action-gh-release@v2 with: name: ${{ steps.release_name.outputs.name }} tag_name: ${{ github.ref_name }} body_path: ${{ steps.release_notes.outputs.path }} files: dist/** - overwrite_files: true - make_latest: ${{ env.SIGN_MACOS == 'true' && !contains(steps.release_name.outputs.name, '-') }} # Mark as prerelease only when the version has a suffix after x.y.z # (e.g. -alpha, -beta). Otherwise publish a normal release. prerelease: ${{ contains(steps.release_name.outputs.name, '-') }} - - name: Clean up signed promotion handoff assets - if: ${{ env.RELEASE_MODE == 'promote_signed' }} - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - - release_id="$(gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${GITHUB_REF_NAME}" --jq '.id')" - gh api --paginate "repos/${GITHUB_REPOSITORY}/releases/${release_id}/assets" \ - --jq '.[] | [.id, .name] | @tsv' | - while IFS=$'\t' read -r asset_id asset_name; do - if [[ -z "$asset_id" || -z "$asset_name" ]]; then - continue - fi - - delete_asset=false - if [[ "$asset_name" == *unsigned* || "$asset_name" == "$SIGNED_MACOS_ASSET" ]]; then - delete_asset=true - fi - - if [[ "$delete_asset" == "true" ]]; then - echo "Deleting release asset ${asset_name}" - gh api -X DELETE "repos/${GITHUB_REPOSITORY}/releases/assets/${asset_id}" - fi - done - - - if: ${{ env.SIGN_MACOS == 'true' }} - uses: facebook/dotslash-publish-release@9c9ec027515c34db9282a09a25a9cab5880b2c52 # v2 + - if: github.repository_owner == 'openai' + uses: facebook/dotslash-publish-release@v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: tag: ${{ github.ref_name }} config: .github/dotslash-config.json - - if: ${{ env.SIGN_MACOS == 'true' }} - uses: facebook/dotslash-publish-release@9c9ec027515c34db9282a09a25a9cab5880b2c52 # v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag: ${{ github.ref_name }} - config: .github/dotslash-zsh-config.json - - - if: ${{ env.SIGN_MACOS == 'true' }} - uses: facebook/dotslash-publish-release@9c9ec027515c34db9282a09a25a9cab5880b2c52 # v2 + - name: Trigger developers.openai.com deploy + # Only trigger the deploy if the release is not a pre-release. + # The deploy is used to update the developers.openai.com website with the new config schema json file. + if: ${{ !contains(steps.release_name.outputs.name, '-') && github.repository_owner == 'openai' }} + continue-on-error: true env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag: ${{ github.ref_name }} - config: .github/dotslash-argument-comment-lint-config.json + DEV_WEBSITE_VERCEL_DEPLOY_HOOK_URL: ${{ secrets.DEV_WEBSITE_VERCEL_DEPLOY_HOOK_URL }} + run: | + if ! curl -sS -f -o /dev/null -X POST "$DEV_WEBSITE_VERCEL_DEPLOY_HOOK_URL"; then + echo "::warning title=developers.openai.com deploy hook failed::Vercel deploy hook POST failed for ${GITHUB_REF_NAME}" + exit 1 + fi # Publish to npm using OIDC authentication. # July 31, 2025: https://github.blog/changelog/2025-07-31-npm-trusted-publishing-with-oidc-is-generally-available/ # npm docs: https://docs.npmjs.com/trusted-publishers publish-npm: # Publish to npm for stable releases and alpha pre-releases with numeric suffixes. - # promote_signed intentionally skips build jobs that are ancestors of release; - # include the !cancelled() status function so Actions does not apply its implicit - # success() check to the whole dependency chain before evaluating release outputs. - if: >- - ${{ - !cancelled() && - needs.release.result == 'success' && - needs.release.outputs.should_publish_npm == 'true' - }} + if: ${{ needs.release.outputs.should_publish_npm == 'true' && github.repository_owner == 'openai' }} name: publish-npm needs: release runs-on: ubuntu-latest @@ -1763,37 +1166,36 @@ jobs: steps: - name: Setup Node.js - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + uses: actions/setup-node@v6 with: - # Node 24 bundles npm >= 11.5.1, which trusted publishing requires. - node-version: 24 + node-version: 22 registry-url: "https://registry.npmjs.org" scope: "@openai" + # Trusted publishing requires npm CLI version 11.5.1 or later. + - name: Update npm + run: npm install -g npm@latest + - name: Download npm tarballs from release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - RELEASE_TAG: ${{ needs.release.outputs.tag }} - RELEASE_VERSION: ${{ needs.release.outputs.version }} run: | set -euo pipefail - version="$RELEASE_VERSION" - tag="$RELEASE_TAG" + version="${{ needs.release.outputs.version }}" + tag="${{ needs.release.outputs.tag }}" mkdir -p dist/npm - patterns=( - "codex-npm-${version}.tgz" - "codex-npm-linux-*-${version}.tgz" - "codex-npm-darwin-*-${version}.tgz" - "codex-npm-win32-*-${version}.tgz" - "codex-responses-api-proxy-npm-${version}.tgz" - "codex-sdk-npm-${version}.tgz" - ) - for pattern in "${patterns[@]}"; do - gh release download "$tag" \ - --repo "${GITHUB_REPOSITORY}" \ - --pattern "$pattern" \ - --dir dist/npm - done + gh release download "$tag" \ + --repo "${GITHUB_REPOSITORY}" \ + --pattern "codex-npm-${version}.tgz" \ + --dir dist/npm + gh release download "$tag" \ + --repo "${GITHUB_REPOSITORY}" \ + --pattern "codex-responses-api-proxy-npm-${version}.tgz" \ + --dir dist/npm + gh release download "$tag" \ + --repo "${GITHUB_REPOSITORY}" \ + --pattern "codex-sdk-npm-${version}.tgz" \ + --dir dist/npm # No NODE_AUTH_TOKEN needed because we use OIDC. - name: Publish to npm @@ -1802,176 +1204,23 @@ jobs: NPM_TAG: ${{ needs.release.outputs.npm_tag }} run: | set -euo pipefail - prefix="" + tag_args=() if [[ -n "${NPM_TAG}" ]]; then - prefix="${NPM_TAG}-" + tag_args+=(--tag "${NPM_TAG}") fi - root_tarball="dist/npm/codex-npm-${VERSION}.tgz" - sdk_tarball="dist/npm/codex-sdk-npm-${VERSION}.tgz" - # Keep this list in sync with CODEX_PLATFORM_PACKAGES in - # codex-cli/scripts/build_npm_package.py. The root wrapper advances - # @openai/codex@latest as soon as it publishes, so every platform - # package it aliases must already exist in the registry first. - platform_tarballs=( - "dist/npm/codex-npm-linux-x64-${VERSION}.tgz" - "dist/npm/codex-npm-linux-arm64-${VERSION}.tgz" - "dist/npm/codex-npm-darwin-x64-${VERSION}.tgz" - "dist/npm/codex-npm-darwin-arm64-${VERSION}.tgz" - "dist/npm/codex-npm-win32-x64-${VERSION}.tgz" - "dist/npm/codex-npm-win32-arm64-${VERSION}.tgz" - ) - - for required_tarball in "${platform_tarballs[@]}" "${root_tarball}"; do - if [[ ! -f "${required_tarball}" ]]; then - echo "Missing npm tarball: ${required_tarball}" - exit 1 - fi - done - - shopt -s nullglob - other_tarballs=() - for tarball in dist/npm/*-"${VERSION}".tgz; do - if [[ "${tarball}" == "${root_tarball}" || "${tarball}" == "${sdk_tarball}" ]]; then - continue - fi - - is_platform_tarball=false - for platform_tarball in "${platform_tarballs[@]}"; do - if [[ "${tarball}" == "${platform_tarball}" ]]; then - is_platform_tarball=true - break - fi - done - if [[ "${is_platform_tarball}" == true ]]; then - continue - fi - - other_tarballs+=("${tarball}") - done - - # Publish the platform packages before the root CLI wrapper. The root - # wrapper advances @openai/codex@latest, so it should only publish - # after the optional dependency versions it references exist. tarballs=( - "${platform_tarballs[@]}" - "${other_tarballs[@]}" - "${root_tarball}" + "codex-npm-${VERSION}.tgz" + "codex-responses-api-proxy-npm-${VERSION}.tgz" + "codex-sdk-npm-${VERSION}.tgz" ) - if [[ -f "${sdk_tarball}" ]]; then - tarballs+=("${sdk_tarball}") - fi for tarball in "${tarballs[@]}"; do - filename="$(basename "${tarball}")" - tag="" - - case "${filename}" in - codex-npm-linux-*-"${VERSION}".tgz|codex-npm-darwin-*-"${VERSION}".tgz|codex-npm-win32-*-"${VERSION}".tgz) - platform="${filename#codex-npm-}" - platform="${platform%-${VERSION}.tgz}" - tag="${prefix}${platform}" - ;; - codex-npm-"${VERSION}".tgz|codex-responses-api-proxy-npm-"${VERSION}".tgz|codex-sdk-npm-"${VERSION}".tgz) - tag="${NPM_TAG}" - ;; - *) - echo "Unexpected npm tarball: ${filename}" - exit 1 - ;; - esac - - publish_cmd=(npm publish "${GITHUB_WORKSPACE}/${tarball}") - if [[ -n "${tag}" ]]; then - publish_cmd+=(--tag "${tag}") - fi - - echo "+ ${publish_cmd[*]}" - set +e - publish_output="$("${publish_cmd[@]}" 2>&1)" - publish_status=$? - set -e - - echo "${publish_output}" - if [[ ${publish_status} -eq 0 ]]; then - continue - fi - - if grep -qiE "previously published|cannot publish over|version already exists" <<< "${publish_output}"; then - echo "Skipping already-published package version for ${filename}" - continue - fi - - exit "${publish_status}" + npm publish "${GITHUB_WORKSPACE}/dist/npm/${tarball}" "${tag_args[@]}" done - deploy-dev-website: - name: Trigger developers.openai.com deploy - needs: release - # Only trigger the deploy for a stable signed release. - # The deploy updates developers.openai.com with the new config schema json file. - if: >- - ${{ - !cancelled() && - needs.release.result == 'success' && - needs.release.outputs.sign_macos == 'true' && - !contains(needs.release.outputs.version, '-') - }} - runs-on: ubuntu-latest - continue-on-error: true - permissions: {} - environment: - name: dev-website-vercel-deploy - deployment: false - - steps: - - name: Trigger developers.openai.com deploy - continue-on-error: true - env: - DEV_WEBSITE_VERCEL_DEPLOY_HOOK_URL: ${{ secrets.DEV_WEBSITE_VERCEL_DEPLOY_HOOK_URL }} - run: | - if ! curl -sS -f -o /dev/null -X POST "$DEV_WEBSITE_VERCEL_DEPLOY_HOOK_URL"; then - echo "::warning title=developers.openai.com deploy hook failed::Vercel deploy hook POST failed for ${GITHUB_REF_NAME}" - exit 1 - fi - - winget: - name: winget - needs: release - # Only publish stable/mainline releases to WinGet; pre-releases include a - # '-' in the semver string (e.g., 1.2.3-alpha.1). - if: >- - ${{ - !cancelled() && - needs.release.result == 'success' && - needs.release.outputs.sign_macos == 'true' && - !contains(needs.release.outputs.version, '-') - }} - # This job only invokes a GitHub Action to open/update the winget-pkgs PR; - # it does not execute Windows-only tooling, so Linux is sufficient. - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - name: Publish to WinGet - uses: vedantmgoyal9/winget-releaser@7bd472be23763def6e16bd06cc8b1cdfab0e2fd5 - with: - identifier: OpenAI.Codex - version: ${{ needs.release.outputs.version }} - release-tag: ${{ needs.release.outputs.tag }} - fork-user: openai-oss-forks - installers-regex: '^codex-(?:x86_64|aarch64)-pc-windows-msvc\.exe\.zip$' - token: ${{ secrets.WINGET_PUBLISH_PAT }} - update-branch: name: Update latest-alpha-cli branch - if: >- - ${{ - !cancelled() && - needs.release.result == 'success' && - needs.release.outputs.sign_macos == 'true' - }} permissions: contents: write needs: release diff --git a/.github/workflows/rusty-v8-release.yml b/.github/workflows/rusty-v8-release.yml index d6fb73e96d8..3b56a1e93f6 100644 --- a/.github/workflows/rusty-v8-release.yml +++ b/.github/workflows/rusty-v8-release.yml @@ -5,11 +5,6 @@ on: tags: - "rusty-v8-v*.*.*" -# Cargo's libgit2 transport has been flaky when fetching git dependencies with -# nested submodules. Prefer the system git CLI for Cargo smoke tests. -env: - CARGO_NET_GIT_FETCH_WITH_CLI: "true" - concurrency: group: ${{ github.workflow }}::${{ github.ref_name }} cancel-in-progress: false @@ -157,9 +152,9 @@ jobs: python-version: "3.12" - name: Set up Rust toolchain for Cargo smoke - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + uses: dtolnay/rust-toolchain@a0b273b48ed29de4470960879e8381ff45632f26 # 1.93.0 with: - toolchain: "1.95.0" + toolchain: "1.93.0" - name: Build Bazel V8 release pair env: @@ -191,10 +186,11 @@ jobs: bazel_args+=(--config=v8-release-compat) fi - ./.github/scripts/run_bazel_with_buildbuddy.py \ + bazel \ --noexperimental_remote_repo_contents_cache \ "${bazel_args[@]}" \ - "--config=${{ matrix.bazel_config }}" + "--config=${{ matrix.bazel_config }}" \ + "--remote_header=x-buildbuddy-api-key=${BUILDBUDDY_API_KEY}" - name: Stage release pair env: diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml index 4103a948cd0..54ed8dc558e 100644 --- a/.github/workflows/sdk.yml +++ b/.github/workflows/sdk.yml @@ -23,15 +23,15 @@ jobs: run: | set -euo pipefail - # Run inside a glibc Linux image so dependency resolution exercises - # the pinned manylinux runtime wheel that users install. + # Run inside Alpine so dependency resolution exercises the pinned + # runtime wheel on the same Linux wheel family that CI installs. docker run --rm \ --user "$(id -u):$(id -g)" \ -e HOME=/tmp/codex-python-sdk-home \ -e UV_LINK_MODE=copy \ -v "${GITHUB_WORKSPACE}:${GITHUB_WORKSPACE}" \ -w "${GITHUB_WORKSPACE}/sdk/python" \ - python:3.12-slim \ + python:3.12-alpine \ sh -euxc ' python -m venv /tmp/uv /tmp/uv/bin/python -m pip install uv==0.11.3 diff --git a/.github/workflows/shell-tool-mcp.yml b/.github/workflows/shell-tool-mcp.yml new file mode 100644 index 00000000000..66a76aa4d93 --- /dev/null +++ b/.github/workflows/shell-tool-mcp.yml @@ -0,0 +1,461 @@ +name: shell-tool-mcp + +on: + workflow_call: + inputs: + release-version: + description: Version to publish (x.y.z or x.y.z-alpha.N). Defaults to GITHUB_REF_NAME when it starts with rust-v. + required: false + type: string + release-tag: + description: Tag name to use when downloading release artifacts (defaults to rust-v). + required: false + type: string + publish: + description: Whether to publish to npm when the version is releasable. + required: false + default: true + type: boolean + +env: + NODE_VERSION: 22 + +jobs: + metadata: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.compute.outputs.version }} + release_tag: ${{ steps.compute.outputs.release_tag }} + should_publish: ${{ steps.compute.outputs.should_publish }} + npm_tag: ${{ steps.compute.outputs.npm_tag }} + steps: + - name: Compute version and tags + id: compute + run: | + set -euo pipefail + + version="${{ inputs.release-version }}" + release_tag="${{ inputs.release-tag }}" + + if [[ -z "$version" ]]; then + if [[ -n "$release_tag" && "$release_tag" =~ ^rust-v.+ ]]; then + version="${release_tag#rust-v}" + elif [[ "${GITHUB_REF_NAME:-}" =~ ^rust-v.+ ]]; then + version="${GITHUB_REF_NAME#rust-v}" + release_tag="${GITHUB_REF_NAME}" + else + echo "release-version is required when GITHUB_REF_NAME is not a rust-v tag." + exit 1 + fi + fi + + if [[ -z "$release_tag" ]]; then + release_tag="rust-v${version}" + fi + + npm_tag="" + should_publish="false" + if [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + should_publish="true" + elif [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+-alpha\.[0-9]+$ ]]; then + should_publish="true" + npm_tag="alpha" + fi + + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "release_tag=${release_tag}" >> "$GITHUB_OUTPUT" + echo "npm_tag=${npm_tag}" >> "$GITHUB_OUTPUT" + echo "should_publish=${should_publish}" >> "$GITHUB_OUTPUT" + + matrix-setup: + runs-on: ubuntu-latest + outputs: + rust-binaries: ${{ steps.compute.outputs.rust-binaries }} + bash-linux: ${{ steps.compute.outputs.bash-linux }} + bash-darwin: ${{ steps.compute.outputs.bash-darwin }} + steps: + - name: Compute matrices + id: compute + shell: bash + run: | + set -euo pipefail + is_openai="${{ github.repository_owner == 'openai' }}" + + # rust-binaries: always include x86_64-musl; conditionally include paid/fork runners + if [[ "$is_openai" == "true" ]]; then + rust='[ + {"runner":"macos-15-xlarge","target":"aarch64-apple-darwin"}, + {"runner":"macos-15-xlarge","target":"x86_64-apple-darwin"}, + {"runner":"ubuntu-24.04-arm","target":"aarch64-unknown-linux-musl","install_musl":true} + ]' + else + rust='[{"runner":"macos-latest","target":"aarch64-apple-darwin"}]' + fi + rust=$(echo "$rust" | jq -c '. + [{"runner":"ubuntu-24.04","target":"x86_64-unknown-linux-musl","install_musl":true}]') + echo "rust-binaries={\"include\":$rust}" >> "$GITHUB_OUTPUT" + + # bash-linux: always include x86_64 variants; add arm64 on openai + bash_linux='[ + {"runner":"ubuntu-24.04","target":"x86_64-unknown-linux-musl","variant":"ubuntu-24.04","image":"ubuntu:24.04"}, + {"runner":"ubuntu-24.04","target":"x86_64-unknown-linux-musl","variant":"ubuntu-22.04","image":"ubuntu:22.04"}, + {"runner":"ubuntu-24.04","target":"x86_64-unknown-linux-musl","variant":"debian-12","image":"debian:12"}, + {"runner":"ubuntu-24.04","target":"x86_64-unknown-linux-musl","variant":"debian-11","image":"debian:11"}, + {"runner":"ubuntu-24.04","target":"x86_64-unknown-linux-musl","variant":"centos-9","image":"quay.io/centos/centos:stream9"} + ]' + if [[ "$is_openai" == "true" ]]; then + bash_linux=$(echo "$bash_linux" | jq -c '. + [ + {"runner":"ubuntu-24.04-arm","target":"aarch64-unknown-linux-musl","variant":"ubuntu-24.04","image":"arm64v8/ubuntu:24.04"}, + {"runner":"ubuntu-24.04-arm","target":"aarch64-unknown-linux-musl","variant":"ubuntu-22.04","image":"arm64v8/ubuntu:22.04"}, + {"runner":"ubuntu-24.04-arm","target":"aarch64-unknown-linux-musl","variant":"ubuntu-20.04","image":"arm64v8/ubuntu:20.04"}, + {"runner":"ubuntu-24.04-arm","target":"aarch64-unknown-linux-musl","variant":"debian-12","image":"arm64v8/debian:12"}, + {"runner":"ubuntu-24.04-arm","target":"aarch64-unknown-linux-musl","variant":"debian-11","image":"arm64v8/debian:11"}, + {"runner":"ubuntu-24.04-arm","target":"aarch64-unknown-linux-musl","variant":"centos-9","image":"quay.io/centos/centos:stream9"} + ]') + fi + echo "bash-linux={\"include\":$(echo "$bash_linux" | jq -c)}" >> "$GITHUB_OUTPUT" + + # bash-darwin: always include macos-14; add macos-15-xlarge on openai + bash_darwin='[{"runner":"macos-14","target":"aarch64-apple-darwin","variant":"macos-14"}]' + if [[ "$is_openai" == "true" ]]; then + bash_darwin=$(echo "$bash_darwin" | jq -c '. + [ + {"runner":"macos-15-xlarge","target":"aarch64-apple-darwin","variant":"macos-15"} + ]') + fi + echo "bash-darwin={\"include\":$(echo "$bash_darwin" | jq -c)}" >> "$GITHUB_OUTPUT" + + rust-binaries: + name: Build Rust - ${{ matrix.target }} + needs: [metadata, matrix-setup] + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + defaults: + run: + working-directory: codex-rs + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix-setup.outputs.rust-binaries) }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Install UBSan runtime (musl) + if: ${{ matrix.install_musl }} + shell: bash + run: | + set -euo pipefail + if command -v apt-get >/dev/null 2>&1; then + sudo apt-get update -y + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libubsan1 + fi + + - uses: dtolnay/rust-toolchain@1.93 + with: + targets: ${{ matrix.target }} + + - if: ${{ matrix.install_musl }} + name: Install Zig + uses: mlugg/setup-zig@v2 + with: + version: 0.14.0 + + - if: ${{ matrix.install_musl }} + name: Install musl build dependencies + env: + TARGET: ${{ matrix.target }} + run: bash "${GITHUB_WORKSPACE}/.github/scripts/install-musl-build-tools.sh" + + - if: ${{ matrix.install_musl }} + name: Configure rustc UBSan wrapper (musl host) + shell: bash + run: | + set -euo pipefail + ubsan="" + if command -v ldconfig >/dev/null 2>&1; then + ubsan="$(ldconfig -p | grep -m1 'libubsan\.so\.1' | sed -E 's/.*=> (.*)$/\1/')" + fi + wrapper_root="${RUNNER_TEMP:-/tmp}" + wrapper="${wrapper_root}/rustc-ubsan-wrapper" + cat > "${wrapper}" <> "$GITHUB_ENV" + echo "RUSTC_WORKSPACE_WRAPPER=" >> "$GITHUB_ENV" + + - if: ${{ matrix.install_musl }} + name: Clear sanitizer flags (musl) + shell: bash + run: | + set -euo pipefail + # Clear global Rust flags so host/proc-macro builds don't pull in UBSan. + echo "RUSTFLAGS=" >> "$GITHUB_ENV" + echo "CARGO_ENCODED_RUSTFLAGS=" >> "$GITHUB_ENV" + echo "RUSTDOCFLAGS=" >> "$GITHUB_ENV" + # Override any runner-level Cargo config rustflags as well. + echo "CARGO_BUILD_RUSTFLAGS=" >> "$GITHUB_ENV" + echo "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS=" >> "$GITHUB_ENV" + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS=" >> "$GITHUB_ENV" + echo "CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS=" >> "$GITHUB_ENV" + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS=" >> "$GITHUB_ENV" + + sanitize_flags() { + local input="$1" + input="${input//-fsanitize=undefined/}" + input="${input//-fno-sanitize-recover=undefined/}" + input="${input//-fno-sanitize-trap=undefined/}" + echo "$input" + } + + cflags="$(sanitize_flags "${CFLAGS-}")" + cxxflags="$(sanitize_flags "${CXXFLAGS-}")" + echo "CFLAGS=${cflags}" >> "$GITHUB_ENV" + echo "CXXFLAGS=${cxxflags}" >> "$GITHUB_ENV" + + - name: Build exec server binaries + run: cargo build --release --target ${{ matrix.target }} --bin codex-exec-mcp-server --bin codex-execve-wrapper + + - name: Stage exec server binaries + run: | + dest="${GITHUB_WORKSPACE}/artifacts/vendor/${{ matrix.target }}" + mkdir -p "$dest" + cp "target/${{ matrix.target }}/release/codex-exec-mcp-server" "$dest/" + cp "target/${{ matrix.target }}/release/codex-execve-wrapper" "$dest/" + + - uses: actions/upload-artifact@v6 + with: + name: shell-tool-mcp-rust-${{ matrix.target }} + path: artifacts/** + if-no-files-found: error + + bash-linux: + name: Build Bash (Linux) - ${{ matrix.variant }} - ${{ matrix.target }} + needs: [metadata, matrix-setup] + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + container: + image: ${{ matrix.image }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix-setup.outputs.bash-linux) }} + steps: + - name: Install build prerequisites + shell: bash + run: | + set -euo pipefail + if command -v apt-get >/dev/null 2>&1; then + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y git build-essential bison autoconf gettext + elif command -v dnf >/dev/null 2>&1; then + dnf install -y git gcc gcc-c++ make bison autoconf gettext + elif command -v yum >/dev/null 2>&1; then + yum install -y git gcc gcc-c++ make bison autoconf gettext + else + echo "Unsupported package manager in container" + exit 1 + fi + + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Build patched Bash + shell: bash + run: | + set -euo pipefail + git clone --depth 1 https://github.com/bolinfest/bash /tmp/bash + cd /tmp/bash + git fetch --depth 1 origin a8a1c2fac029404d3f42cd39f5a20f24b6e4fe4b + git checkout a8a1c2fac029404d3f42cd39f5a20f24b6e4fe4b + git apply "${GITHUB_WORKSPACE}/shell-tool-mcp/patches/bash-exec-wrapper.patch" + ./configure --without-bash-malloc + cores="$(command -v nproc >/dev/null 2>&1 && nproc || getconf _NPROCESSORS_ONLN)" + make -j"${cores}" + + dest="${GITHUB_WORKSPACE}/artifacts/vendor/${{ matrix.target }}/bash/${{ matrix.variant }}" + mkdir -p "$dest" + cp bash "$dest/bash" + + - uses: actions/upload-artifact@v6 + with: + name: shell-tool-mcp-bash-${{ matrix.target }}-${{ matrix.variant }} + path: artifacts/** + if-no-files-found: error + + bash-darwin: + name: Build Bash (macOS) - ${{ matrix.variant }} - ${{ matrix.target }} + needs: [metadata, matrix-setup] + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix-setup.outputs.bash-darwin) }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Build patched Bash + shell: bash + run: | + set -euo pipefail + git clone --depth 1 https://github.com/bolinfest/bash /tmp/bash + cd /tmp/bash + git fetch --depth 1 origin a8a1c2fac029404d3f42cd39f5a20f24b6e4fe4b + git checkout a8a1c2fac029404d3f42cd39f5a20f24b6e4fe4b + git apply "${GITHUB_WORKSPACE}/shell-tool-mcp/patches/bash-exec-wrapper.patch" + ./configure --without-bash-malloc + cores="$(getconf _NPROCESSORS_ONLN)" + make -j"${cores}" + + dest="${GITHUB_WORKSPACE}/artifacts/vendor/${{ matrix.target }}/bash/${{ matrix.variant }}" + mkdir -p "$dest" + cp bash "$dest/bash" + + - uses: actions/upload-artifact@v6 + with: + name: shell-tool-mcp-bash-${{ matrix.target }}-${{ matrix.variant }} + path: artifacts/** + if-no-files-found: error + + package: + name: Package npm module + needs: + - metadata + - rust-binaries + - bash-linux + - bash-darwin + runs-on: ubuntu-latest + env: + PACKAGE_VERSION: ${{ needs.metadata.outputs.version }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + run_install: false + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Install JavaScript dependencies + run: pnpm install --frozen-lockfile + + - name: Build (shell-tool-mcp) + run: pnpm --filter @openai/codex-shell-tool-mcp run build + + - name: Download build artifacts + uses: actions/download-artifact@v7 + with: + path: artifacts + + - name: Assemble staging directory + id: staging + shell: bash + run: | + set -euo pipefail + staging="${STAGING_DIR}" + mkdir -p "$staging" "$staging/vendor" + cp shell-tool-mcp/README.md "$staging/" + cp shell-tool-mcp/package.json "$staging/" + cp -R shell-tool-mcp/bin "$staging/" + + found_vendor="false" + shopt -s nullglob + for vendor_dir in artifacts/*/vendor; do + rsync -av "$vendor_dir/" "$staging/vendor/" + found_vendor="true" + done + if [[ "$found_vendor" == "false" ]]; then + echo "No vendor payloads were downloaded." + exit 1 + fi + + node - <<'NODE' + import fs from "node:fs"; + import path from "node:path"; + + const stagingDir = process.env.STAGING_DIR; + const version = process.env.PACKAGE_VERSION; + const pkgPath = path.join(stagingDir, "package.json"); + const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8")); + pkg.version = version; + fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + "\n"); + NODE + + echo "dir=$staging" >> "$GITHUB_OUTPUT" + env: + STAGING_DIR: ${{ runner.temp }}/shell-tool-mcp + + - name: Ensure binaries are executable + run: | + set -euo pipefail + staging="${{ steps.staging.outputs.dir }}" + chmod +x \ + "$staging"/vendor/*/codex-exec-mcp-server \ + "$staging"/vendor/*/codex-execve-wrapper \ + "$staging"/vendor/*/bash/*/bash + + - name: Create npm tarball + shell: bash + run: | + set -euo pipefail + mkdir -p dist/npm + staging="${{ steps.staging.outputs.dir }}" + pack_info=$(cd "$staging" && npm pack --ignore-scripts --json --pack-destination "${GITHUB_WORKSPACE}/dist/npm") + filename=$(PACK_INFO="$pack_info" node -e 'const data = JSON.parse(process.env.PACK_INFO); console.log(data[0].filename);') + mv "dist/npm/${filename}" "dist/npm/codex-shell-tool-mcp-npm-${PACKAGE_VERSION}.tgz" + + - uses: actions/upload-artifact@v6 + with: + name: codex-shell-tool-mcp-npm + path: dist/npm/codex-shell-tool-mcp-npm-${{ env.PACKAGE_VERSION }}.tgz + if-no-files-found: error + + publish: + name: Publish npm package + needs: + - metadata + - package + if: ${{ inputs.publish && needs.metadata.outputs.should_publish == 'true' }} + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: ${{ env.NODE_VERSION }} + registry-url: https://registry.npmjs.org + scope: "@openai" + + # Trusted publishing requires npm CLI version 11.5.1 or later. + - name: Update npm + run: npm install -g npm@latest + + - name: Download npm tarball + uses: actions/download-artifact@v7 + with: + name: codex-shell-tool-mcp-npm + path: dist/npm + + - name: Publish to npm + if: github.repository == 'openai/codex' + env: + NPM_TAG: ${{ needs.metadata.outputs.npm_tag }} + VERSION: ${{ needs.metadata.outputs.version }} + shell: bash + run: | + set -euo pipefail + tag_args=() + if [[ -n "${NPM_TAG}" ]]; then + tag_args+=(--tag "${NPM_TAG}") + fi + npm publish "dist/npm/codex-shell-tool-mcp-npm-${VERSION}.tgz" "${tag_args[@]}" diff --git a/.github/workflows/termux-release-checkpoint.yml b/.github/workflows/termux-release-checkpoint.yml new file mode 100644 index 00000000000..0347e6f1b3d --- /dev/null +++ b/.github/workflows/termux-release-checkpoint.yml @@ -0,0 +1,103 @@ +name: termux-release-checkpoint + +on: + workflow_dispatch: + inputs: + source_branch: + description: "Release branch to checkpoint from, for example release/0.123.0" + required: false + type: string + default: "" + source_sha: + description: "Specific source commit SHA to checkpoint; defaults to the source branch tip" + required: false + type: string + default: "" + destination_branch: + description: "Destination patch branch to receive the checkpoint PR" + required: false + type: string + default: "wallentx/termux-target" + reviewer: + description: "GitHub username to request as reviewer" + required: false + type: string + default: "wallentx" + +permissions: + actions: read + attestations: read + checks: read + contents: read + deployments: read + issues: read + discussions: read + packages: read + pages: read + pull-requests: read + repository-projects: read + statuses: read + +concurrency: + group: termux-release-checkpoint-${{ inputs.source_branch || github.ref_name }} + cancel-in-progress: false + +defaults: + run: + shell: bash + +jobs: + checkpoint: + runs-on: ubuntu-slim + permissions: + contents: write + issues: write + pull-requests: write + env: + DESTINATION_BRANCH: ${{ inputs.destination_branch || 'wallentx/termux-target' }} + REVIEWER: ${{ inputs.reviewer || 'wallentx' }} + REQUESTED_SOURCE_BRANCH: ${{ inputs.source_branch }} + REQUESTED_SOURCE_SHA: ${{ inputs.source_sha }} + TERMUX_AUTOMATION_DIR: ${{ github.workspace }}/.termux-release-automation + steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ vars.APP_CLIENT_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + + - name: Export GitHub App token for gh + env: + APP_TOKEN: ${{ steps.app-token.outputs.token }} + run: echo "GH_TOKEN=${APP_TOKEN}" >> "${GITHUB_ENV}" + + - name: Checkout release branch + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ inputs.source_branch || github.ref_name }} + token: ${{ steps.app-token.outputs.token }} + + - name: Checkout automation helpers + uses: actions/checkout@v6 + with: + fetch-depth: 1 + ref: ${{ github.workflow_sha }} + path: .termux-release-automation + token: ${{ steps.app-token.outputs.token }} + + - name: 🧰 Actions Toolbox + uses: wallentx/gh-actions/composite/actions-toolbox@main + + - name: Validate GitHub CLI environment + run: bash "${TERMUX_AUTOMATION_DIR}/scripts/termux-validate-gh-env.sh" + + - name: Configure git + run: | + set -euo pipefail + source_branch="${REQUESTED_SOURCE_BRANCH:-${GITHUB_REF_NAME}}" + bash "${TERMUX_AUTOMATION_DIR}/scripts/termux-configure-git.sh" --origin "${DESTINATION_BRANCH}" "${source_branch}" + + - name: Create checkpoint PR + run: bash "${TERMUX_AUTOMATION_DIR}/scripts/termux-create-checkpoint-pr.sh" diff --git a/.github/workflows/termux-release-deploy.yml b/.github/workflows/termux-release-deploy.yml new file mode 100644 index 00000000000..2071611dacc --- /dev/null +++ b/.github/workflows/termux-release-deploy.yml @@ -0,0 +1,243 @@ +name: termux-release-deploy + +on: + push: + branches: + - "release/**" + workflow_dispatch: + inputs: + release_branch: + description: "Release branch to deploy, for example release/0.124.0" + required: true + type: string + release_sha: + description: "Release branch commit SHA to deploy. Defaults to the branch head." + required: false + default: "" + type: string + pr_number: + description: "Merged release PR number to promote. Optional; normally discovered automatically." + required: false + default: "" + type: string + pr_head_sha: + description: "Merged release PR head SHA. Required only when pr_number is set." + required: false + default: "" + type: string + destination_branch: + description: "Destination patch branch to receive the checkpoint PR" + required: false + default: "wallentx/termux-target" + type: string + reviewer: + description: "GitHub username to request as reviewer on the checkpoint PR" + required: false + default: "wallentx" + type: string + +permissions: + actions: read + attestations: read + checks: read + contents: read + deployments: read + issues: read + discussions: read + packages: read + pages: read + pull-requests: read + repository-projects: read + statuses: read + +concurrency: + group: termux-release-deploy-${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.ref_name }} + cancel-in-progress: false + +defaults: + run: + shell: bash + +jobs: + deploy: + runs-on: ubuntu-24.04 + if: ${{ github.event_name == 'workflow_dispatch' || !startsWith(github.event.head_commit.message, 'Seed Termux release automation') }} + permissions: + actions: read + contents: write + deployments: write + issues: write + pull-requests: write + env: + REQUESTED_RELEASE_BRANCH: ${{ inputs.release_branch }} + REQUESTED_RELEASE_SHA: ${{ inputs.release_sha }} + INPUT_PR_NUMBER: ${{ inputs.pr_number }} + INPUT_PR_HEAD_SHA: ${{ inputs.pr_head_sha }} + DESTINATION_BRANCH: ${{ inputs.destination_branch || 'wallentx/termux-target' }} + REVIEWER: ${{ inputs.reviewer || 'wallentx' }} + TERMUX_AUTOMATION_DIR: ${{ github.workspace }}/.termux-release-automation + steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ vars.APP_CLIENT_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + + - name: Export GitHub App token for gh + env: + APP_TOKEN: ${{ steps.app-token.outputs.token }} + run: echo "GH_TOKEN=${APP_TOKEN}" >> "${GITHUB_ENV}" + + - name: Checkout release branch + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.ref }} + token: ${{ steps.app-token.outputs.token }} + + - name: Checkout automation helpers + uses: actions/checkout@v6 + with: + fetch-depth: 1 + ref: ${{ github.workflow_sha }} + path: .termux-release-automation + token: ${{ steps.app-token.outputs.token }} + + - name: 🧰 Actions Toolbox + uses: wallentx/gh-actions/composite/actions-toolbox@main + + - name: Validate GitHub CLI environment + run: bash "${TERMUX_AUTOMATION_DIR}/scripts/termux-validate-gh-env.sh" + + - name: Configure git + run: | + set -euo pipefail + release_branch="${REQUESTED_RELEASE_BRANCH:-${GITHUB_REF_NAME}}" + bash "${TERMUX_AUTOMATION_DIR}/scripts/termux-configure-git.sh" --origin "${DESTINATION_BRANCH}" "${release_branch}" + + - name: Resolve release ref + id: release-ref + run: bash "${TERMUX_AUTOMATION_DIR}/scripts/termux-resolve-release-ref.sh" + + - name: Read release metadata + id: metadata + env: + TERMUX_RELEASE_ACTION: deploy + run: bash "${TERMUX_AUTOMATION_DIR}/scripts/termux-read-release-metadata.sh" + + - name: Create deployment + if: steps.metadata.outputs.deploy == 'true' + id: deployment + env: + GH_TOKEN: ${{ github.token }} + RELEASE_SHA: ${{ steps.release-ref.outputs.sha }} + TERMUX_TAG: ${{ steps.metadata.outputs.termux_tag }} + run: | + set -euo pipefail + deployment_id="$( + gh api \ + -X POST \ + "repos/${GITHUB_REPOSITORY}/deployments" \ + -f ref="${RELEASE_SHA}" \ + -f environment="termux-release" \ + -F auto_merge=false \ + -F required_contexts[] \ + -f description="Termux release deployment for ${TERMUX_TAG}" \ + --jq '.id' + )" + echo "id=${deployment_id}" >> "$GITHUB_OUTPUT" + + - name: Mark deployment in progress + if: steps.metadata.outputs.deploy == 'true' + env: + GH_TOKEN: ${{ github.token }} + DEPLOYMENT_ID: ${{ steps.deployment.outputs.id }} + run: | + set -euo pipefail + log_url="${GH_WORKFLOW_URL:-${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}}" + gh api \ + -X POST \ + "repos/${GITHUB_REPOSITORY}/deployments/${DEPLOYMENT_ID}/statuses" \ + -f state="in_progress" \ + -f environment="termux-release" \ + -f log_url="${log_url}" \ + -F auto_inactive=false \ + -f description="Promoting Termux release artifact and preparing checkpoint PR" \ + >/dev/null + + - name: Locate merged pull request + if: steps.metadata.outputs.deploy == 'true' && steps.metadata.outputs.asset_exists != 'true' + id: pr + env: + RELEASE_BRANCH: ${{ steps.release-ref.outputs.branch }} + RELEASE_SHA: ${{ steps.release-ref.outputs.sha }} + run: bash "${TERMUX_AUTOMATION_DIR}/scripts/termux-find-release-pr.sh" + + - name: Download promoted PR artifact + if: steps.metadata.outputs.deploy == 'true' && steps.metadata.outputs.asset_exists != 'true' + env: + PR_ARTIFACT_NAME: ${{ steps.pr.outputs.artifact_name }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + run: bash "${TERMUX_AUTOMATION_DIR}/scripts/termux-download-release-artifact.sh" + + - name: Create or update mirrored Termux release + if: steps.metadata.outputs.deploy == 'true' && steps.metadata.outputs.asset_exists != 'true' + env: + UPSTREAM_TAG: ${{ steps.metadata.outputs.upstream_tag }} + UPSTREAM_REPO: ${{ steps.metadata.outputs.upstream_repo }} + UPSTREAM_NAME: ${{ steps.metadata.outputs.upstream_name }} + TERMUX_TAG: ${{ steps.metadata.outputs.termux_tag }} + UPSTREAM_PRERELEASE: ${{ steps.metadata.outputs.upstream_prerelease }} + UPSTREAM_HTML_URL: ${{ steps.metadata.outputs.upstream_html_url }} + RELEASE_TRAIN: ${{ steps.metadata.outputs.release_train }} + RELEASE_EXISTS: ${{ steps.metadata.outputs.release_exists }} + PR_NUMBER: ${{ steps.pr.outputs.number }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + RELEASE_SHA: ${{ steps.release-ref.outputs.sha }} + run: bash "${TERMUX_AUTOMATION_DIR}/scripts/termux-create-or-update-mirrored-release.sh" + + - name: Ensure checkpoint PR + if: steps.metadata.outputs.deploy == 'true' + id: checkpoint + env: + SOURCE_BRANCH: ${{ steps.release-ref.outputs.branch }} + SOURCE_SHA: ${{ steps.release-ref.outputs.sha }} + run: bash "${TERMUX_AUTOMATION_DIR}/scripts/termux-create-checkpoint-pr.sh" + + - name: Mark deployment success + if: steps.metadata.outputs.deploy == 'true' + env: + GH_TOKEN: ${{ github.token }} + DEPLOYMENT_ID: ${{ steps.deployment.outputs.id }} + TERMUX_TAG: ${{ steps.metadata.outputs.termux_tag }} + run: | + set -euo pipefail + log_url="${GH_WORKFLOW_URL:-${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}}" + gh api \ + -X POST \ + "repos/${GITHUB_REPOSITORY}/deployments/${DEPLOYMENT_ID}/statuses" \ + -f state="success" \ + -f environment="termux-release" \ + -f log_url="${log_url}" \ + -F auto_inactive=false \ + -f description="Termux release deployment completed for ${TERMUX_TAG}" \ + >/dev/null + + - name: Mark deployment failure + if: failure() && steps.deployment.outputs.id != '' + env: + GH_TOKEN: ${{ github.token }} + DEPLOYMENT_ID: ${{ steps.deployment.outputs.id }} + run: | + set -euo pipefail + log_url="${GH_WORKFLOW_URL:-${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}}" + gh api \ + -X POST \ + "repos/${GITHUB_REPOSITORY}/deployments/${DEPLOYMENT_ID}/statuses" \ + -f state="failure" \ + -f environment="termux-release" \ + -f log_url="${log_url}" \ + -F auto_inactive=false \ + -f description="Termux release deployment failed" \ + >/dev/null diff --git a/.github/workflows/termux-release-promote.yml b/.github/workflows/termux-release-promote.yml new file mode 100644 index 00000000000..22c277c21a7 --- /dev/null +++ b/.github/workflows/termux-release-promote.yml @@ -0,0 +1,135 @@ +name: termux-release-promote + +on: + workflow_dispatch: + inputs: + release_branch: + description: "Release branch to promote, for example release/0.122.0" + required: true + type: string + release_sha: + description: "Release branch commit SHA to promote. Defaults to the branch head." + required: false + default: "" + type: string + pr_number: + description: "Merged PR number to promote. Optional; normally discovered automatically." + required: false + default: "" + type: string + pr_head_sha: + description: "Merged PR head SHA. Required only when pr_number is set." + required: false + default: "" + type: string + +permissions: + actions: read + attestations: read + checks: read + contents: read + deployments: read + issues: read + discussions: read + packages: read + pages: read + pull-requests: read + repository-projects: read + statuses: read + +concurrency: + group: termux-release-promote-${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.ref_name }} + cancel-in-progress: false + +defaults: + run: + shell: bash + +jobs: + promote: + runs-on: ubuntu-24.04 + permissions: + actions: read + contents: write + pull-requests: read + env: + TERMUX_AUTOMATION_DIR: ${{ github.workspace }}/.termux-release-automation + steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ vars.APP_CLIENT_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + + - name: Export GitHub App token for gh + env: + APP_TOKEN: ${{ steps.app-token.outputs.token }} + run: echo "GH_TOKEN=${APP_TOKEN}" >> "${GITHUB_ENV}" + + - name: Checkout release branch + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.ref }} + token: ${{ steps.app-token.outputs.token }} + + - name: Checkout automation helpers + uses: actions/checkout@v6 + with: + fetch-depth: 1 + ref: ${{ github.workflow_sha }} + path: .termux-release-automation + token: ${{ steps.app-token.outputs.token }} + + - name: 🧰 Actions Toolbox + uses: wallentx/gh-actions/composite/actions-toolbox@main + + - name: Validate GitHub CLI environment + run: bash "${TERMUX_AUTOMATION_DIR}/scripts/termux-validate-gh-env.sh" + + - name: Resolve release ref + id: release-ref + env: + INPUT_RELEASE_BRANCH: ${{ inputs.release_branch }} + INPUT_RELEASE_SHA: ${{ inputs.release_sha }} + run: bash "${TERMUX_AUTOMATION_DIR}/scripts/termux-resolve-release-ref.sh" + + - name: Read release metadata + id: metadata + env: + TERMUX_RELEASE_ACTION: promote + run: bash "${TERMUX_AUTOMATION_DIR}/scripts/termux-read-release-metadata.sh" + + - name: Locate merged pull request + if: steps.metadata.outputs.promote == 'true' + id: pr + env: + RELEASE_BRANCH: ${{ steps.release-ref.outputs.branch }} + RELEASE_SHA: ${{ steps.release-ref.outputs.sha }} + INPUT_PR_NUMBER: ${{ inputs.pr_number }} + INPUT_PR_HEAD_SHA: ${{ inputs.pr_head_sha }} + run: bash "${TERMUX_AUTOMATION_DIR}/scripts/termux-find-release-pr.sh" + + - name: Download promoted PR artifact + if: steps.metadata.outputs.promote == 'true' + env: + PR_ARTIFACT_NAME: ${{ steps.pr.outputs.artifact_name }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + run: bash "${TERMUX_AUTOMATION_DIR}/scripts/termux-download-release-artifact.sh" + + - name: Create or update mirrored Termux release + if: steps.metadata.outputs.promote == 'true' + env: + UPSTREAM_TAG: ${{ steps.metadata.outputs.upstream_tag }} + UPSTREAM_REPO: ${{ steps.metadata.outputs.upstream_repo }} + UPSTREAM_NAME: ${{ steps.metadata.outputs.upstream_name }} + TERMUX_TAG: ${{ steps.metadata.outputs.termux_tag }} + UPSTREAM_PRERELEASE: ${{ steps.metadata.outputs.upstream_prerelease }} + UPSTREAM_HTML_URL: ${{ steps.metadata.outputs.upstream_html_url }} + RELEASE_TRAIN: ${{ steps.metadata.outputs.release_train }} + RELEASE_EXISTS: ${{ steps.metadata.outputs.release_exists }} + PR_NUMBER: ${{ steps.pr.outputs.number }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + RELEASE_SHA: ${{ steps.release-ref.outputs.sha }} + run: bash "${TERMUX_AUTOMATION_DIR}/scripts/termux-create-or-update-mirrored-release.sh" diff --git a/.github/workflows/v8-canary.yml b/.github/workflows/v8-canary.yml index 0ad9f850d11..979e991504b 100644 --- a/.github/workflows/v8-canary.yml +++ b/.github/workflows/v8-canary.yml @@ -5,7 +5,6 @@ on: paths: - ".bazelrc" - ".github/actions/setup-bazel-ci/**" - - ".github/scripts/run_bazel_with_buildbuddy.py" - ".github/scripts/rusty_v8_bazel.py" - ".github/scripts/rusty_v8_module_bazel.py" - ".github/workflows/rusty-v8-release.yml" @@ -24,7 +23,6 @@ on: paths: - ".bazelrc" - ".github/actions/setup-bazel-ci/**" - - ".github/scripts/run_bazel_with_buildbuddy.py" - ".github/scripts/rusty_v8_bazel.py" - ".github/scripts/rusty_v8_module_bazel.py" - ".github/workflows/rusty-v8-release.yml" @@ -39,11 +37,6 @@ on: - "third_party/v8/**" workflow_dispatch: -# Cargo's libgit2 transport has been flaky when fetching git dependencies with -# nested submodules. Prefer the system git CLI for Cargo builds and smoke tests. -env: - CARGO_NET_GIT_FETCH_WITH_CLI: "true" - concurrency: group: ${{ github.workflow }}::${{ github.event.pull_request.number > 0 && format('pr-{0}', github.event.pull_request.number) || github.ref_name }} cancel-in-progress: ${{ github.ref_name != 'main' }} @@ -173,9 +166,9 @@ jobs: python-version: "3.12" - name: Set up Rust toolchain for Cargo smoke - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + uses: dtolnay/rust-toolchain@a0b273b48ed29de4470960879e8381ff45632f26 # 1.93.0 with: - toolchain: "1.95.0" + toolchain: "1.93.0" - name: Build Bazel V8 release pair env: @@ -205,10 +198,11 @@ jobs: bazel_args+=(--config=v8-release-compat) fi - ./.github/scripts/run_bazel_with_buildbuddy.py \ + bazel \ --noexperimental_remote_repo_contents_cache \ "${bazel_args[@]}" \ - "--config=${{ matrix.bazel_config }}" + "--config=${{ matrix.bazel_config }}" \ + "--remote_header=x-buildbuddy-api-key=${BUILDBUDDY_API_KEY}" - name: Stage release pair env: @@ -316,9 +310,9 @@ jobs: architecture: x64 - name: Set up Codex Rust toolchain for Cargo smoke - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + uses: dtolnay/rust-toolchain@a0b273b48ed29de4470960879e8381ff45632f26 # 1.93.0 with: - toolchain: "1.95.0" + toolchain: "1.93.0" targets: ${{ matrix.target }} - name: Install rusty_v8 Rust toolchain @@ -407,7 +401,7 @@ jobs: cd codex-rs RUSTY_V8_ARCHIVE="${GITHUB_WORKSPACE}/${archive}" \ RUSTY_V8_SRC_BINDING_PATH="${GITHUB_WORKSPACE}/${binding}" \ - cargo +1.95.0 test -p codex-v8-poc --target "${TARGET}" --features sandbox --no-run + cargo +1.93.0 test -p codex-v8-poc --target "${TARGET}" --features sandbox --no-run ) - name: Upload staged artifacts diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 15e0e2c0102..b6c924c9f36 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -84,6 +84,7 @@ members = [ "v8-poc", "utils/absolute-path", "utils/cargo-bin", + "utils/file-lock", "git-utils", "utils/cache", "utils/image", @@ -120,7 +121,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.0.0" +version = "0.138.0-alpha.4" # Track the edition for all workspace crates in one place. Individual # crates can still override this value, but keeping it here means new # crates created with `cargo new -w ...` automatically inherit the 2024 @@ -226,6 +227,7 @@ codex-utils-cache = { path = "utils/cache" } codex-utils-cargo-bin = { path = "utils/cargo-bin" } codex-utils-cli = { path = "utils/cli" } codex-utils-elapsed = { path = "utils/elapsed" } +codex-utils-file-lock = { path = "utils/file-lock" } codex-utils-fuzzy-match = { path = "utils/fuzzy-match" } codex-utils-home-dir = { path = "utils/home-dir" } codex-utils-image = { path = "utils/image" } diff --git a/codex-rs/arg0/Cargo.toml b/codex-rs/arg0/Cargo.toml index 55526b4d064..c806f011aa6 100644 --- a/codex-rs/arg0/Cargo.toml +++ b/codex-rs/arg0/Cargo.toml @@ -21,6 +21,7 @@ codex-linux-sandbox = { workspace = true } codex-sandboxing = { workspace = true } codex-shell-escalation = { workspace = true } codex-utils-absolute-path = { workspace = true } +codex-utils-file-lock = { workspace = true } codex-utils-home-dir = { workspace = true } dotenvy = { workspace = true } tempfile = { workspace = true } diff --git a/codex-rs/arg0/src/lib.rs b/codex-rs/arg0/src/lib.rs index ba254d57abf..512c9666213 100644 --- a/codex-rs/arg0/src/lib.rs +++ b/codex-rs/arg0/src/lib.rs @@ -8,6 +8,8 @@ use codex_apply_patch::CODEX_CORE_APPLY_PATCH_ARG1; use codex_exec_server::CODEX_FS_HELPER_ARG1; use codex_install_context::InstallContext; use codex_sandboxing::landlock::CODEX_LINUX_SANDBOX_ARG0; +use codex_utils_file_lock::TryFileLockOutcome; +use codex_utils_file_lock::try_lock_exclusive_optional; use codex_utils_home_dir::find_codex_home; #[cfg(unix)] use std::os::unix::fs::symlink; @@ -35,12 +37,12 @@ pub struct Arg0DispatchPaths { /// Keeps the per-session PATH entry alive and locked for the process lifetime. pub struct Arg0PathEntryGuard { _temp_dir: TempDir, - _lock_file: File, + _lock_file: Option, paths: Arg0DispatchPaths, } impl Arg0PathEntryGuard { - fn new(temp_dir: TempDir, lock_file: File, paths: Arg0DispatchPaths) -> Self { + fn new(temp_dir: TempDir, lock_file: Option, paths: Arg0DispatchPaths) -> Self { Self { _temp_dir: temp_dir, _lock_file: lock_file, @@ -364,7 +366,13 @@ fn prepare_path_entry_for_codex_aliases( .create(true) .truncate(false) .open(&lock_path)?; - lock_file.try_lock()?; + let lock_file = match try_lock_exclusive_optional(&lock_file)? { + TryFileLockOutcome::Acquired => Some(lock_file), + TryFileLockOutcome::Unsupported => None, + TryFileLockOutcome::WouldBlock => { + return Err(std::io::Error::from(std::io::ErrorKind::WouldBlock).into()); + } + }; for filename in &[ APPLY_PATCH_ARG0, @@ -497,10 +505,9 @@ fn try_lock_dir(dir: &Path) -> std::io::Result> { Err(err) => return Err(err), }; - match lock_file.try_lock() { - Ok(()) => Ok(Some(lock_file)), - Err(std::fs::TryLockError::WouldBlock) => Ok(None), - Err(err) => Err(err.into()), + match try_lock_exclusive_optional(&lock_file)? { + TryFileLockOutcome::Acquired => Ok(Some(lock_file)), + TryFileLockOutcome::WouldBlock | TryFileLockOutcome::Unsupported => Ok(None), } } @@ -715,7 +722,13 @@ mod tests { let dir = root.path().join("locked"); fs::create_dir(&dir)?; let lock_file = create_lock(&dir)?; - lock_file.try_lock()?; + match try_lock_exclusive_optional(&lock_file)? { + TryFileLockOutcome::Acquired => {} + TryFileLockOutcome::Unsupported => return Ok(()), + TryFileLockOutcome::WouldBlock => { + panic!("newly created lock file should not be locked"); + } + } janitor_cleanup(root.path())?; diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 97e104464fa..d95c4f70f70 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -66,6 +66,7 @@ codex-thread-store = { workspace = true } codex-tools = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-cache = { workspace = true } +codex-utils-file-lock = { workspace = true } codex-utils-image = { workspace = true } codex-utils-home-dir = { workspace = true } codex-utils-output-truncation = { workspace = true } @@ -127,6 +128,10 @@ openssl-sys = { workspace = true, features = ["vendored"] } [target.aarch64-unknown-linux-musl.dependencies] openssl-sys = { workspace = true, features = ["vendored"] } +# Build OpenSSL from source for Android builds. +[target.aarch64-linux-android.dependencies] +openssl-sys = { workspace = true, features = ["vendored"] } + [target.'cfg(unix)'.dependencies] codex-shell-escalation = { workspace = true } diff --git a/codex-rs/core/src/installation_id.rs b/codex-rs/core/src/installation_id.rs index a42e6b6d835..a87c660f613 100644 --- a/codex-rs/core/src/installation_id.rs +++ b/codex-rs/core/src/installation_id.rs @@ -11,6 +11,9 @@ use std::os::unix::fs::OpenOptionsExt; use std::os::unix::fs::PermissionsExt; use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_file_lock::FileLockOutcome; +use codex_utils_file_lock::acquire_sibling_lock_dir; +use codex_utils_file_lock::lock_exclusive_optional; use tokio::fs; use uuid::Uuid; @@ -29,7 +32,10 @@ pub async fn resolve_installation_id(codex_home: &AbsolutePathBuf) -> Result None, + FileLockOutcome::Unsupported => Some(acquire_sibling_lock_dir(&path)?), + }; #[cfg(unix)] { diff --git a/codex-rs/execpolicy/Cargo.toml b/codex-rs/execpolicy/Cargo.toml index b22226a79e4..b87af084d7d 100644 --- a/codex-rs/execpolicy/Cargo.toml +++ b/codex-rs/execpolicy/Cargo.toml @@ -21,6 +21,7 @@ workspace = true anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } codex-utils-absolute-path = { workspace = true } +codex-utils-file-lock = { workspace = true } multimap = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } diff --git a/codex-rs/execpolicy/src/amend.rs b/codex-rs/execpolicy/src/amend.rs index e25fd1bd914..1ef36b1ff5d 100644 --- a/codex-rs/execpolicy/src/amend.rs +++ b/codex-rs/execpolicy/src/amend.rs @@ -6,6 +6,10 @@ use std::io::Write; use std::path::Path; use std::path::PathBuf; +use codex_utils_file_lock::FileLockOutcome; +use codex_utils_file_lock::acquire_sibling_lock_dir; +use codex_utils_file_lock::lock_exclusive_optional; + use crate::decision::Decision; use crate::rule::NetworkRuleProtocol; use crate::rule::normalize_network_rule_host; @@ -154,10 +158,21 @@ fn append_locked_line(policy_path: &Path, line: &str) -> Result<(), AmendError> path: policy_path.to_path_buf(), source, })?; - file.lock().map_err(|source| AmendError::LockPolicyFile { - path: policy_path.to_path_buf(), - source, - })?; + let _lock_dir_guard = + match lock_exclusive_optional(&file).map_err(|source| AmendError::LockPolicyFile { + path: policy_path.to_path_buf(), + source, + })? { + FileLockOutcome::Acquired => None, + FileLockOutcome::Unsupported => { + Some(acquire_sibling_lock_dir(policy_path).map_err(|source| { + AmendError::LockPolicyFile { + path: policy_path.to_path_buf(), + source, + } + })?) + } + }; file.seek(SeekFrom::Start(0)) .map_err(|source| AmendError::SeekPolicyFile { diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 581f67bf94f..b4b1933f7c4 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -125,7 +125,7 @@ uuid = { workspace = true } codex-windows-sandbox = { workspace = true } tokio-util = { workspace = true, features = ["time"] } -[target.'cfg(not(target_os = "linux"))'.dependencies] +[target.'cfg(all(not(target_os = "linux"), not(target_os = "android")))'.dependencies] cpal = "0.15" [target.'cfg(unix)'.dependencies] diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index b53ec0f83da..7d57c26d383 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -94,9 +94,9 @@ mod app_server_approval_conversions; mod app_server_session; mod approval_events; mod ascii_animation; -#[cfg(not(target_os = "linux"))] +#[cfg(not(any(target_os = "linux", target_os = "android")))] mod audio_device; -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "android"))] #[allow(dead_code)] mod audio_device { use crate::app_event::RealtimeAudioDeviceKind; @@ -203,11 +203,11 @@ mod update_prompt; mod update_versions; mod updates; mod version; -#[cfg(not(target_os = "linux"))] +#[cfg(not(any(target_os = "linux", target_os = "android")))] mod voice; mod width; mod workspace_command; -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "android"))] #[allow(dead_code)] mod voice { use crate::app_event_sender::AppEventSender; diff --git a/codex-rs/utils/file-lock/BUILD.bazel b/codex-rs/utils/file-lock/BUILD.bazel new file mode 100644 index 00000000000..face70c5351 --- /dev/null +++ b/codex-rs/utils/file-lock/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "file-lock", + crate_name = "codex_utils_file_lock", +) diff --git a/codex-rs/utils/file-lock/Cargo.toml b/codex-rs/utils/file-lock/Cargo.toml new file mode 100644 index 00000000000..5e3877fc163 --- /dev/null +++ b/codex-rs/utils/file-lock/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "codex-utils-file-lock" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lints] +workspace = true + +[dependencies] diff --git a/codex-rs/utils/file-lock/src/lib.rs b/codex-rs/utils/file-lock/src/lib.rs new file mode 100644 index 00000000000..fdcfbdb81e7 --- /dev/null +++ b/codex-rs/utils/file-lock/src/lib.rs @@ -0,0 +1,168 @@ +use std::fs::File; +use std::fs::create_dir; +use std::fs::remove_dir; +use std::io; +use std::path::Path; +use std::path::PathBuf; +use std::thread; +use std::time::Duration; + +const LOCK_DIR_RETRY_SLEEP: Duration = Duration::from_millis(100); + +/// Result of acquiring a blocking advisory file lock. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FileLockOutcome { + Acquired, + Unsupported, +} + +/// Result of attempting to acquire a non-blocking advisory file lock. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TryFileLockOutcome { + Acquired, + WouldBlock, + Unsupported, +} + +/// Result of attempting to acquire a non-blocking lock directory. +#[derive(Debug, PartialEq, Eq)] +pub enum TryLockDirOutcome { + Acquired(LockDirGuard), + WouldBlock, +} + +/// Guard for a sibling lock directory created with an atomic `mkdir`. +#[derive(Debug, PartialEq, Eq)] +pub struct LockDirGuard { + path: PathBuf, +} + +impl Drop for LockDirGuard { + fn drop(&mut self) { + let _ = remove_dir(&self.path); + } +} + +/// Acquires an exclusive advisory file lock, treating unsupported file locking +/// as a distinct outcome for platforms such as Termux. +pub fn lock_exclusive_optional(file: &File) -> io::Result { + match file.lock() { + Ok(()) => Ok(FileLockOutcome::Acquired), + Err(err) if err.kind() == io::ErrorKind::Unsupported => Ok(FileLockOutcome::Unsupported), + Err(err) => Err(err), + } +} + +/// Attempts to acquire an exclusive advisory file lock without blocking, +/// preserving `WouldBlock` and unsupported file locking as distinct outcomes. +pub fn try_lock_exclusive_optional(file: &File) -> io::Result { + match file.try_lock() { + Ok(()) => Ok(TryFileLockOutcome::Acquired), + Err(std::fs::TryLockError::WouldBlock) => Ok(TryFileLockOutcome::WouldBlock), + Err(std::fs::TryLockError::Error(err)) if err.kind() == io::ErrorKind::Unsupported => { + Ok(TryFileLockOutcome::Unsupported) + } + Err(std::fs::TryLockError::Error(err)) => Err(err), + } +} + +/// Returns the sibling directory path used as a fallback lock for `path`. +pub fn sibling_lock_dir(path: &Path) -> PathBuf { + let Some(file_name) = path.file_name() else { + return path.with_file_name(".lock"); + }; + + let mut lock_name = file_name.to_os_string(); + lock_name.push(".lock"); + path.with_file_name(lock_name) +} + +/// Acquires a sibling lock directory, blocking until it is available. +pub fn acquire_sibling_lock_dir(path: &Path) -> io::Result { + loop { + match try_acquire_sibling_lock_dir(path)? { + TryLockDirOutcome::Acquired(guard) => return Ok(guard), + TryLockDirOutcome::WouldBlock => thread::sleep(LOCK_DIR_RETRY_SLEEP), + } + } +} + +/// Attempts to acquire a sibling lock directory without blocking. +pub fn try_acquire_sibling_lock_dir(path: &Path) -> io::Result { + let lock_dir = sibling_lock_dir(path); + match create_dir(&lock_dir) { + Ok(()) => Ok(TryLockDirOutcome::Acquired(LockDirGuard { path: lock_dir })), + Err(err) if err.kind() == io::ErrorKind::AlreadyExists => Ok(TryLockDirOutcome::WouldBlock), + Err(err) => Err(err), + } +} + +/// Attempts to acquire a shared advisory file lock without blocking, +/// preserving `WouldBlock` and unsupported file locking as distinct outcomes. +pub fn try_lock_shared_optional(file: &File) -> io::Result { + match file.try_lock_shared() { + Ok(()) => Ok(TryFileLockOutcome::Acquired), + Err(std::fs::TryLockError::WouldBlock) => Ok(TryFileLockOutcome::WouldBlock), + Err(std::fs::TryLockError::Error(err)) if err.kind() == io::ErrorKind::Unsupported => { + Ok(TryFileLockOutcome::Unsupported) + } + Err(std::fs::TryLockError::Error(err)) => Err(err), + } +} + +#[cfg(test)] +mod tests { + use super::TryLockDirOutcome; + use super::sibling_lock_dir; + use super::try_acquire_sibling_lock_dir; + use std::fs; + use std::path::PathBuf; + use std::time::SystemTime; + + fn unique_temp_file_path(name: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("system clock should be after Unix epoch") + .as_nanos(); + std::env::temp_dir().join(format!( + "codex-file-lock-{name}-{}-{nanos}", + std::process::id() + )) + } + + #[test] + fn sibling_lock_dir_appends_lock_suffix() { + let path = PathBuf::from("/tmp/history.jsonl"); + + assert_eq!( + sibling_lock_dir(&path), + PathBuf::from("/tmp/history.jsonl.lock") + ); + } + + #[test] + fn try_acquire_sibling_lock_dir_is_exclusive_until_drop() { + let path = unique_temp_file_path("exclusive"); + let lock_dir = sibling_lock_dir(&path); + let _ = fs::remove_dir_all(&lock_dir); + + let guard = match try_acquire_sibling_lock_dir(&path).expect("acquire lock dir") { + TryLockDirOutcome::Acquired(guard) => guard, + TryLockDirOutcome::WouldBlock => panic!("first lock attempt should acquire"), + }; + assert!(lock_dir.is_dir()); + + assert!(matches!( + try_acquire_sibling_lock_dir(&path).expect("try acquire held lock dir"), + TryLockDirOutcome::WouldBlock + )); + + drop(guard); + assert!(!lock_dir.exists()); + + let reacquired = try_acquire_sibling_lock_dir(&path).expect("reacquire lock dir"); + assert!(matches!(reacquired, TryLockDirOutcome::Acquired(_))); + + let _ = fs::remove_dir_all(lock_dir); + } +} diff --git a/justfile b/justfile index fe7e7349b31..1ab4a97a1d1 100644 --- a/justfile +++ b/justfile @@ -168,6 +168,11 @@ argument-comment-lint *args: argument-comment-lint-from-source *args: {{ python }} {{ justfile_directory() }}/tools/argument-comment-lint/run.py {args} +# Audit advisory file locks that may need Termux Unsupported handling. +[no-cd] +termux-lock-audit *args: + {{ justfile_directory() }}/scripts/termux-lock-audit.sh "$@" + # Tail logs from the state SQLite database [unix] log *args: diff --git a/scripts/termux-configure-git.sh b/scripts/termux-configure-git.sh new file mode 100755 index 00000000000..fae3abcbab4 --- /dev/null +++ b/scripts/termux-configure-git.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# Configure the explicit bot identity used by release automation and fetch any +# refs requested by the calling workflow. + +set -euo pipefail + +git config user.name "github-actions[bot]" +git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + +while (($#)); do + case "$1" in + --origin) + shift + origin_refs=() + while (($#)) && [[ "$1" != --* ]]; do + origin_refs+=("$1") + shift + done + if ((${#origin_refs[@]})); then + git fetch --prune origin "${origin_refs[@]}" + fi + ;; + --upstream-tag) + if (($# < 3)); then + echo "--upstream-tag requires ." >&2 + exit 1 + fi + upstream_repo="$2" + upstream_tag="$3" + git remote add upstream "https://github.com/${upstream_repo}.git" 2>/dev/null || true + git fetch --prune --no-tags upstream "+refs/tags/${upstream_tag}:refs/tags/${upstream_tag}" + shift 3 + ;; + *) + echo "Unknown argument: $1" >&2 + exit 1 + ;; + esac +done diff --git a/scripts/termux-create-checkpoint-pr.sh b/scripts/termux-create-checkpoint-pr.sh new file mode 100755 index 00000000000..f315a5b9182 --- /dev/null +++ b/scripts/termux-create-checkpoint-pr.sh @@ -0,0 +1,275 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/termux-release-paths.sh +source "${script_dir}/termux-release-paths.sh" + +source_branch="${SOURCE_BRANCH:-${REQUESTED_SOURCE_BRANCH:-${GITHUB_REF_NAME}}}" +source_sha="${SOURCE_SHA:-${REQUESTED_SOURCE_SHA:-}}" +if [[ -z "${source_sha}" ]]; then + if [[ "${GITHUB_EVENT_NAME:-}" == "push" && "${source_branch}" == "${GITHUB_REF_NAME:-}" ]]; then + source_sha="${GITHUB_SHA}" + else + source_sha="$(git rev-parse "origin/${source_branch}")" + fi +fi + +if [[ -z "${DESTINATION_BRANCH:-}" ]]; then + echo "DESTINATION_BRANCH is required." >&2 + exit 1 +fi + +release_only_checkpoint_paths() { + printf '%s\n' "${TERMUX_RELEASE_BRANCH_SCRIPT_PATHS[@]}" +} + +resolve_source_version_conflicts() { + local path="$1" + local resolved_path + resolved_path="$(mktemp)" + + if ! awk ' + function normalize_versions(text) { + gsub(/version = "[^"]+"/, "version = \"\"", text) + return text + } + + BEGIN { + in_block = 0 + side = "" + ours = "" + theirs = "" + blocks = 0 + } + + /^<<<<<<< / { + if (in_block) { + exit 1 + } + in_block = 1 + side = "ours" + ours = "" + theirs = "" + blocks++ + next + } + + /^=======$/ && in_block { + side = "theirs" + next + } + + /^>>>>>>> / && in_block { + if (normalize_versions(ours) != normalize_versions(theirs)) { + exit 1 + } + printf "%s", theirs + in_block = 0 + side = "" + next + } + + { + if (!in_block) { + print + } else if (side == "ours") { + ours = ours $0 ORS + } else if (side == "theirs") { + theirs = theirs $0 ORS + } else { + exit 1 + } + } + + END { + if (in_block || blocks == 0) { + exit 1 + } + } + ' "${path}" > "${resolved_path}"; then + rm -f "${resolved_path}" + return 1 + fi + + cp "${resolved_path}" "${path}" + rm -f "${resolved_path}" +} + +short_sha="${source_sha:0:12}" +source_slug="${source_branch//\//_}" +dest_slug="${DESTINATION_BRANCH//\//_}" +checkpoint_branch="checkpoint/${dest_slug}_from_${source_slug}_${short_sha}" +pr_title="checkpoint: into ${DESTINATION_BRANCH} from ${source_branch} @ ${short_sha}" +merge_conflicted=false +conflict_summary="" + +existing_pr="$( + gh pr list \ + --repo "${GITHUB_REPOSITORY}" \ + --head "${checkpoint_branch}" \ + --state all \ + --json number,state,mergedAt,url \ + --jq '[.[] | select(.state == "OPEN" or .mergedAt != null)] | .[0] // empty' +)" +if [[ -n "${existing_pr}" ]]; then + existing_url="$(jq -r '.url' <<< "${existing_pr}")" + existing_state="$(jq -r '.state' <<< "${existing_pr}")" + echo "Checkpoint PR already exists for ${checkpoint_branch}: ${existing_url} (${existing_state})." + if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo "pr_url=${existing_url}" >> "${GITHUB_OUTPUT}" + fi + exit 0 +fi + +git checkout -B "${checkpoint_branch}" "origin/${DESTINATION_BRANCH}" + +if ! git merge --no-ff --no-edit "${source_sha}"; then + mapfile -t conflicted_paths < <(git diff --name-only --diff-filter=U) + for conflicted_path in "${conflicted_paths[@]}"; do + if termux_is_checkpoint_release_only_path "${conflicted_path}"; then + echo "Auto-resolving release-only checkpoint conflict in ${conflicted_path} by keeping ${DESTINATION_BRANCH}." + if git cat-file -e "HEAD:${conflicted_path}" 2>/dev/null; then + git checkout --ours -- "${conflicted_path}" + git add "${conflicted_path}" + else + git rm -f --ignore-unmatch "${conflicted_path}" + fi + fi + done + + mapfile -t remaining_conflicts < <(git diff --name-only --diff-filter=U) + if ((${#remaining_conflicts[@]})); then + cargo_version_conflicts=true + for remaining_conflict in "${remaining_conflicts[@]}"; do + case "${remaining_conflict}" in + codex-rs/Cargo.toml|codex-rs/Cargo.lock) + ;; + *) + cargo_version_conflicts=false + ;; + esac + done + + if [[ "${cargo_version_conflicts}" == "true" ]]; then + for remaining_conflict in "${remaining_conflicts[@]}"; do + if ! resolve_source_version_conflicts "${remaining_conflict}"; then + cargo_version_conflicts=false + break + fi + done + + if [[ "${cargo_version_conflicts}" == "true" ]]; then + echo "Auto-resolving recurring Cargo version checkpoint conflicts by keeping ${source_branch} versions." + git add -- "${remaining_conflicts[@]}" + fi + fi + fi + + mapfile -t remaining_conflicts < <(git diff --name-only --diff-filter=U) + if [[ "${#remaining_conflicts[@]}" -eq 0 ]]; then + git commit --no-edit + else + merge_conflicted=true + conflict_summary="$( + printf '%s\n' "${remaining_conflicts[@]}" | awk '{ print "- `" $0 "`" }' + )" + echo "Automatic checkpoint merge failed; creating a manual-resolution PR instead." >&2 + if git rev-parse -q --verify MERGE_HEAD >/dev/null; then + git merge --abort + fi + git checkout -B "${checkpoint_branch}" "${source_sha}" + fi +fi + +if git cat-file -e "origin/${DESTINATION_BRANCH}:.github" 2>/dev/null; then + git checkout "origin/${DESTINATION_BRANCH}" -- .github + mapfile -t added_github_paths < <( + git diff --name-only --diff-filter=A "origin/${DESTINATION_BRANCH}" -- .github + ) + if ((${#added_github_paths[@]})); then + git rm -f --ignore-unmatch -- "${added_github_paths[@]}" + fi +else + git rm -r --ignore-unmatch .github +fi + +while IFS= read -r release_only_path; do + if git cat-file -e "origin/${DESTINATION_BRANCH}:${release_only_path}" 2>/dev/null; then + git checkout "origin/${DESTINATION_BRANCH}" -- "${release_only_path}" + else + git rm -f --ignore-unmatch -- "${release_only_path}" + fi +done < <(release_only_checkpoint_paths) + +if ! git diff --quiet || ! git diff --cached --quiet; then + if [[ -e .github || -L .github ]] || git ls-files --error-unmatch -- .github >/dev/null 2>&1; then + git add -A .github + fi + while IFS= read -r release_only_path; do + if [[ -e "${release_only_path}" || -L "${release_only_path}" ]] || git ls-files --error-unmatch -- "${release_only_path}" >/dev/null 2>&1; then + git add -A -- "${release_only_path}" + fi + done < <(release_only_checkpoint_paths) + if [[ "${merge_conflicted}" == "true" ]]; then + git commit -m "checkpoint: prepare ${source_branch} for ${DESTINATION_BRANCH}" + else + git commit --amend --no-edit + fi +fi + +if git diff --quiet "origin/${DESTINATION_BRANCH}" HEAD; then + echo "Checkpoint merge produced no destination changes after release-only files were restored." + exit 0 +fi + +git push --force-with-lease origin "${checkpoint_branch}" + +remaining="$( + git log --first-parent --pretty=format:%H "${source_sha}..origin/${source_branch}" | wc -w +)" + +body_path="${RUNNER_TEMP}/termux-checkpoint-pr.md" +{ + echo "## Termux release checkpoint" + echo + echo "- Source branch: \`${source_branch}\`" + echo "- Source hash: \`${source_sha}\`" + echo "- Destination branch: \`${DESTINATION_BRANCH}\`" + echo "- Remaining first-parent commits on source: ${remaining}" + echo + echo "This PR carries release-train conflict fixes and follow-up changes back into the reusable Termux patch branch." + if [[ "${merge_conflicted}" == "true" ]]; then + echo + echo "## Merge conflicts" + echo + echo "GitHub Actions could not create the checkpoint merge commit automatically, so this PR was created from the source branch state for manual conflict resolution." + echo + echo "Conflicted paths from the failed merge attempt:" + if [[ -n "${conflict_summary}" ]]; then + printf '%s\n' "${conflict_summary}" + else + echo "- Conflict details unavailable" + fi + fi + echo + echo "Release-only workflow files and metadata under \`.github\` were restored to the destination branch versions before opening this PR." +} > "${body_path}" + +pr_url="$( + gh pr create \ + --repo "${GITHUB_REPOSITORY}" \ + --base "${DESTINATION_BRANCH}" \ + --head "${checkpoint_branch}" \ + --title "${pr_title}" \ + --body-file "${body_path}" +)" +gh pr edit "${pr_url}" --repo "${GITHUB_REPOSITORY}" --add-reviewer "${REVIEWER}" || true +gh label create checkpoint --repo "${GITHUB_REPOSITORY}" --color c5def5 --description "Checkpoint merge" --force +gh label create termux-release --repo "${GITHUB_REPOSITORY}" --color 0e8a16 --description "Termux release automation" --force +gh pr edit "${pr_url}" --repo "${GITHUB_REPOSITORY}" --add-label "checkpoint" --add-label "termux-release" + +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo "pr_url=${pr_url}" >> "${GITHUB_OUTPUT}" +fi diff --git a/scripts/termux-create-or-update-mirrored-release.sh b/scripts/termux-create-or-update-mirrored-release.sh new file mode 100755 index 00000000000..5787e894582 --- /dev/null +++ b/scripts/termux-create-or-update-mirrored-release.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +# Create a mirrored Termux release or repair an existing release that is missing +# the Android tarball. + +set -euo pipefail + +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}" +: "${GH_TOKEN:?GH_TOKEN is required}" +: "${RUNNER_TEMP:?RUNNER_TEMP is required}" +: "${UPSTREAM_TAG:?UPSTREAM_TAG is required}" +: "${UPSTREAM_REPO:?UPSTREAM_REPO is required}" +: "${UPSTREAM_NAME?UPSTREAM_NAME is required}" +: "${TERMUX_TAG:?TERMUX_TAG is required}" +: "${UPSTREAM_PRERELEASE:?UPSTREAM_PRERELEASE is required}" +: "${UPSTREAM_HTML_URL?UPSTREAM_HTML_URL is required}" +: "${RELEASE_TRAIN?RELEASE_TRAIN is required}" +: "${RELEASE_EXISTS:?RELEASE_EXISTS is required}" +: "${PR_NUMBER:?PR_NUMBER is required}" +: "${HEAD_SHA:?HEAD_SHA is required}" +: "${RELEASE_SHA:?RELEASE_SHA is required}" + +promoted_dir="${PROMOTED_DIR:-promoted}" +asset_path="${promoted_dir}/codex-aarch64-linux-android.tar.gz" + +body_path="${RUNNER_TEMP}/release-body.md" +upstream_body_path="${RUNNER_TEMP}/upstream-release-body.md" +upstream_body_without_changelog_path="${RUNNER_TEMP}/upstream-release-body-without-changelog.md" +if gh release view "${UPSTREAM_TAG}" \ + --repo "${UPSTREAM_REPO}" \ + --json body \ + --jq '.body // ""' > "${upstream_body_path}"; then + awk ' + function heading_level(line, text) { + if (match(line, /^(#{1,6})[[:space:]]+(.+)$/, parts)) { + text = tolower(parts[2]) + sub(/[[:space:]]+#+[[:space:]]*$/, "", text) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", text) + if (text == "changelog" || text == "change log") { + return length(parts[1]) + } + } + return 0 + } + + { + if (!skip) { + level = heading_level($0) + if (level > 0) { + skip = 1 + skip_level = level + next + } + print + next + } + + if (match($0, /^(#{1,6})[[:space:]]+/, parts) && length(parts[1]) <= skip_level) { + skip = 0 + print + } + } + ' "${upstream_body_path}" > "${upstream_body_without_changelog_path}" +else + echo "::warning title=Upstream release notes unavailable::Could not read ${UPSTREAM_REPO} release ${UPSTREAM_TAG}." + : > "${upstream_body_without_changelog_path}" +fi + +{ + echo "Termux Android build for ${UPSTREAM_TAG}." + echo + echo "- Upstream release: ${UPSTREAM_HTML_URL}" + echo "- Release train: \`${RELEASE_TRAIN}\`" + echo "- Promoted PR: #${PR_NUMBER}" + echo "- Promoted PR head SHA: \`${HEAD_SHA}\`" +} > "${body_path}" +if [[ -s "${upstream_body_without_changelog_path}" ]]; then + { + echo + echo "## Upstream release notes" + echo + cat "${upstream_body_without_changelog_path}" + } >> "${body_path}" +fi + +release_title="${UPSTREAM_NAME}" +if [[ -z "${release_title}" || "${release_title}" == "null" ]]; then + release_title="${TERMUX_TAG}" +fi + +if [[ "${RELEASE_EXISTS}" == "true" ]]; then + gh release upload \ + "${TERMUX_TAG}" \ + "${asset_path}#codex-termux" \ + --repo "${GITHUB_REPOSITORY}" \ + --clobber + exit 0 +fi + +release_args=( + gh release create "${TERMUX_TAG}" + --repo "${GITHUB_REPOSITORY}" + --target "${RELEASE_SHA}" + --title "${release_title}" + --notes-file "${body_path}" +) +if [[ "${UPSTREAM_PRERELEASE}" == "true" ]]; then + release_args+=(--prerelease) +fi +release_args+=("${asset_path}#codex-termux") +"${release_args[@]}" diff --git a/scripts/termux-download-release-artifact.sh b/scripts/termux-download-release-artifact.sh new file mode 100755 index 00000000000..2acdba9025a --- /dev/null +++ b/scripts/termux-download-release-artifact.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ -z "${HEAD_SHA:-}" ]]; then + echo "HEAD_SHA is required." >&2 + exit 1 +fi + +if [[ -z "${PR_ARTIFACT_NAME:-}" ]]; then + echo "PR_ARTIFACT_NAME is required." >&2 + exit 1 +fi + +promoted_dir="${PROMOTED_DIR:-promoted}" + +find_run_id() { + local event="$1" + gh run list \ + --repo "${GITHUB_REPOSITORY}" \ + --workflow rust-release.yml \ + --event "${event}" \ + --status success \ + --commit "${HEAD_SHA}" \ + --limit 1 \ + --json databaseId \ + --jq '.[0].databaseId // empty' +} + +find_artifact_run_id() { + local artifact_name="$1" + + gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${artifact_name}" \ + | jq -rs \ + --arg artifact_name "${artifact_name}" \ + --arg head_sha "${HEAD_SHA}" \ + ' + [ + .[].artifacts[] + | select(.name == $artifact_name) + | select(.expired == false) + | select(.workflow_run.head_sha == $head_sha) + ] + | sort_by(.created_at) + | reverse + | .[0].workflow_run.id // empty + ' +} + +artifact_name="${PR_ARTIFACT_NAME}" +run_id="$(find_artifact_run_id "${artifact_name}")" +mkdir -p "${promoted_dir}" +if [[ -n "${run_id}" ]] && gh run download "${run_id}" \ + --repo "${GITHUB_REPOSITORY}" \ + --name "${artifact_name}" \ + --dir "${promoted_dir}"; then + : +else + artifact_name="aarch64-linux-android" + run_id="$(find_run_id workflow_dispatch)" + if [[ -z "${run_id}" ]]; then + echo "No successful rust-release run found for ${HEAD_SHA}" >&2 + exit 1 + fi + gh run download "${run_id}" \ + --repo "${GITHUB_REPOSITORY}" \ + --name "${artifact_name}" \ + --dir "${promoted_dir}" +fi + +ls -la "${promoted_dir}" +if [[ -f "${promoted_dir}/SHA256SUMS" ]]; then + (cd "${promoted_dir}" && sha256sum -c SHA256SUMS) +fi +if [[ ! -f "${promoted_dir}/codex-aarch64-linux-android.tar.gz" ]]; then + echo "Expected ${promoted_dir}/codex-aarch64-linux-android.tar.gz in the downloaded artifact." >&2 + exit 1 +fi diff --git a/scripts/termux-find-release-pr.sh b/scripts/termux-find-release-pr.sh new file mode 100755 index 00000000000..3e35fa42eab --- /dev/null +++ b/scripts/termux-find-release-pr.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ -n "${INPUT_PR_NUMBER:-}" || -n "${INPUT_PR_HEAD_SHA:-}" ]]; then + if [[ -z "${INPUT_PR_NUMBER:-}" || -z "${INPUT_PR_HEAD_SHA:-}" ]]; then + echo "workflow_dispatch inputs pr_number and pr_head_sha must be provided together." >&2 + exit 1 + fi + + pr_number="${INPUT_PR_NUMBER}" + head_sha="${INPUT_PR_HEAD_SHA}" + head_ref="" +else + if [[ -z "${RELEASE_BRANCH:-}" || -z "${RELEASE_SHA:-}" ]]; then + echo "RELEASE_BRANCH and RELEASE_SHA are required." >&2 + exit 1 + fi + + pr_json="$( + gh pr list \ + --repo "${GITHUB_REPOSITORY}" \ + --base "${RELEASE_BRANCH}" \ + --state merged \ + --limit 100 \ + --json number,headRefOid,headRefName,mergeCommit \ + --jq "[.[] | select(.mergeCommit.oid == \"${RELEASE_SHA}\")] | sort_by(.number) | reverse | .[0] // empty" + )" + if [[ -z "${pr_json}" ]]; then + echo "Unable to find merged PR for ${RELEASE_SHA} into ${RELEASE_BRANCH}" >&2 + exit 1 + fi + + pr_number="$(jq -r '.number' <<< "${pr_json}")" + head_sha="$(jq -r '.headRefOid // .head.sha' <<< "${pr_json}")" + head_ref="$(jq -r '.headRefName // .head.ref' <<< "${pr_json}")" +fi + +artifact_name="termux-android-pr-${pr_number}-${head_sha}" +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + { + echo "number=${pr_number}" + echo "head_sha=${head_sha}" + echo "head_ref=${head_ref}" + echo "artifact_name=${artifact_name}" + } >> "${GITHUB_OUTPUT}" +else + printf 'number=%s\nhead_sha=%s\nhead_ref=%s\nartifact_name=%s\n' \ + "${pr_number}" \ + "${head_sha}" \ + "${head_ref}" \ + "${artifact_name}" +fi diff --git a/scripts/termux-lock-audit.sh b/scripts/termux-lock-audit.sh new file mode 100755 index 00000000000..ac40f8e5724 --- /dev/null +++ b/scripts/termux-lock-audit.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'USAGE' +Usage: scripts/termux-lock-audit.sh [--strict] + +Audits Rust advisory file-lock usage that may need Termux compatibility handling. + +By default this script prints findings and exits successfully. With --strict, it +exits non-zero when candidate file-lock calls are found in files that do not also +mention Unsupported/TryLockError handling. +USAGE +} + +strict=false +for arg in "$@"; do + case "$arg" in + --strict) + strict=true + ;; + -h | --help) + usage + exit 0 + ;; + *) + echo "unknown argument: $arg" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if ! command -v rg >/dev/null 2>&1; then + echo "error: rg is required for this audit" >&2 + exit 1 +fi + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +unsupported_pattern='std::fs::TryLockError|fs::TryLockError|TryLockError::|ErrorKind::Unsupported|std::io::ErrorKind::Unsupported' +lock_context_pattern='(\.lock\(\)|\.try_lock\(|\.try_lock_shared\(|TryLockError|ErrorKind::Unsupported)' +candidate_file_lock_pattern='\b([A-Za-z_][A-Za-z0-9_]*_file|file)\.(lock|try_lock|try_lock_shared)\(' +candidate_false_positive_pattern='poisoned' +helper_pattern='codex_utils_file_lock|FileLockOutcome|TryFileLockOutcome|lock_exclusive_optional|try_lock_exclusive_optional|try_lock_shared_optional' + +print_section() { + printf '\n== %s ==\n' "$1" +} + +print_section "Unsupported-aware lock handling" +echo "These files already mention Unsupported/TryLockError and are likely patched or intentionally reviewed:" + +unsupported_count=0 +while IFS= read -r file; do + if rg -q "$lock_context_pattern" "$file"; then + rg -n -H "$lock_context_pattern" "$file" + unsupported_count=$((unsupported_count + 1)) + fi +done < <(rg -l "$unsupported_pattern" codex-rs -g '*.rs' || true) + +if [ "$unsupported_count" -eq 0 ]; then + echo "No unsupported-aware lock handling found." +fi + +print_section "Optional file-lock helper usage" +echo "These files use the shared optional advisory file-lock helper:" + +helper_count=0 +while IFS= read -r file; do + rg -n -H "$helper_pattern" "$file" + helper_count=$((helper_count + 1)) +done < <(rg -l "$helper_pattern" codex-rs -g '*.rs' || true) + +if [ "$helper_count" -eq 0 ]; then + echo "No optional file-lock helper usage found." +fi + +print_section "Candidate file-lock calls for manual review" +echo "These receiver names may be std::fs::File advisory locks, but the file does not mention Unsupported/TryLockError handling." +echo "This section can include false positives, such as mutexes wrapping a file." + +review_count=0 +while IFS= read -r file; do + if ! rg -q "$unsupported_pattern" "$file"; then + matches="$(rg -n -H "$candidate_file_lock_pattern" "$file" | rg -v "$candidate_false_positive_pattern" || true)" + if [ -n "$matches" ]; then + echo "$matches" + review_count=$((review_count + 1)) + fi + fi +done < <(rg -l "$candidate_file_lock_pattern" codex-rs -g '*.rs' || true) + +if [ "$review_count" -eq 0 ]; then + echo "No unpatched candidate file-lock calls found." +fi + +print_section "Candidate file-lock calls already in unsupported-aware files" +echo "These are candidate file-lock calls in files that already mention Unsupported/TryLockError handling:" + +aware_candidate_count=0 +while IFS= read -r file; do + if rg -q "$unsupported_pattern" "$file"; then + matches="$(rg -n -H "$candidate_file_lock_pattern" "$file" | rg -v "$candidate_false_positive_pattern" || true)" + if [ -n "$matches" ]; then + echo "$matches" + aware_candidate_count=$((aware_candidate_count + 1)) + fi + fi +done < <(rg -l "$candidate_file_lock_pattern" codex-rs -g '*.rs' || true) + +if [ "$aware_candidate_count" -eq 0 ]; then + echo "No unsupported-aware candidate file-lock calls found." +fi + +print_section "Summary" +echo "unsupported-aware files: $unsupported_count" +echo "optional helper files: $helper_count" +echo "review candidate files: $review_count" +echo "unsupported-aware candidate files: $aware_candidate_count" + +if [ "$strict" = true ] && [ "$review_count" -gt 0 ]; then + echo "strict mode: manual-review candidate files were found" >&2 + exit 1 +fi diff --git a/scripts/termux-read-release-metadata.sh b/scripts/termux-read-release-metadata.sh new file mode 100755 index 00000000000..519a0635a36 --- /dev/null +++ b/scripts/termux-read-release-metadata.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# Parse .github/termux-release.json and emit workflow outputs for deploy or +# promote jobs. TERMUX_RELEASE_ACTION must be "deploy" or "promote". + +set -euo pipefail + +: "${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}" +: "${GH_TOKEN:?GH_TOKEN is required}" + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + +metadata=".github/termux-release.json" +action="${TERMUX_RELEASE_ACTION:-deploy}" +case "${action}" in + deploy) + missing_context="deployment" + ;; + promote) + missing_context="promotion" + ;; + *) + echo "TERMUX_RELEASE_ACTION must be deploy or promote." >&2 + exit 1 + ;; +esac + +if [[ ! -f "${metadata}" ]]; then + echo "No ${metadata}; this push is not a Termux release ${missing_context}." + echo "${action}=false" >> "${GITHUB_OUTPUT}" + exit 0 +fi + +upstream_tag="$(jq -r '.upstream_tag // empty' "${metadata}")" +upstream_name="$(jq -r '.upstream_name // .upstream_tag // empty' "${metadata}")" +termux_tag="$(jq -r '.termux_tag // empty' "${metadata}")" +upstream_version="${upstream_tag#rust-v}" +upstream_version="${upstream_version%-termux}" +upstream_prerelease=false +if [[ "${upstream_version}" == *-* ]]; then + upstream_prerelease=true +fi +upstream_html_url="$(jq -r '.upstream_html_url // ""' "${metadata}")" +upstream_repo="$(jq -r '.upstream_repo // "openai/codex"' "${metadata}")" +release_train="$(jq -r '.release_train // ""' "${metadata}")" +if [[ -z "${upstream_tag}" || -z "${termux_tag}" ]]; then + echo "Missing upstream_tag or termux_tag in ${metadata}" >&2 + exit 1 +fi + +release_state="$(TERMUX_TAG="${termux_tag}" "${script_dir}/termux-release-asset-state.sh")" +release_exists="$(awk -F= '$1 == "release_exists" { print $2 }' <<< "${release_state}")" +asset_exists="$(awk -F= '$1 == "asset_exists" { print $2 }' <<< "${release_state}")" + +if [[ "${action}" == "promote" ]]; then + if [[ "${asset_exists}" == "true" ]]; then + echo "${termux_tag} already exists with codex-aarch64-linux-android.tar.gz; skipping promotion." + echo "promote=false" >> "${GITHUB_OUTPUT}" + exit 0 + fi + if [[ "${release_exists}" == "true" ]]; then + echo "${termux_tag} exists but is missing codex-aarch64-linux-android.tar.gz; repairing promotion." + fi +fi + +{ + echo "${action}=true" + echo "upstream_tag=${upstream_tag}" + echo "upstream_name=${upstream_name}" + echo "termux_tag=${termux_tag}" + echo "upstream_prerelease=${upstream_prerelease}" + echo "upstream_html_url=${upstream_html_url}" + echo "upstream_repo=${upstream_repo}" + echo "release_train=${release_train}" + echo "release_exists=${release_exists}" + echo "asset_exists=${asset_exists}" +} >> "${GITHUB_OUTPUT}" diff --git a/scripts/termux-release-asset-state.sh b/scripts/termux-release-asset-state.sh new file mode 100755 index 00000000000..8694480b6a3 --- /dev/null +++ b/scripts/termux-release-asset-state.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -euo pipefail + +termux_tag="${1:-${TERMUX_TAG:-}}" +release_repo="${RELEASE_REPO:-${GITHUB_REPOSITORY:-}}" +asset_name="${ASSET_NAME:-codex-aarch64-linux-android.tar.gz}" + +if [[ -z "${termux_tag}" ]]; then + echo "TERMUX_TAG or tag argument is required." >&2 + exit 1 +fi + +if [[ -z "${release_repo}" ]]; then + echo "GITHUB_REPOSITORY or RELEASE_REPO is required." >&2 + exit 1 +fi + +release_exists=false +asset_exists=false + +if gh release view "${termux_tag}" --repo "${release_repo}" >/dev/null 2>&1; then + release_exists=true + release_asset_exists="$( + gh release view "${termux_tag}" \ + --repo "${release_repo}" \ + --json assets \ + | jq -r --arg asset_name "${asset_name}" \ + '.assets | map(.name) | any(. == $asset_name)' + )" + if [[ "${release_asset_exists}" == "true" ]]; then + asset_exists=true + fi +fi + +printf 'release_exists=%s\n' "${release_exists}" +printf 'asset_exists=%s\n' "${asset_exists}" diff --git a/scripts/termux-release-paths.sh b/scripts/termux-release-paths.sh new file mode 100755 index 00000000000..6990f323ec4 --- /dev/null +++ b/scripts/termux-release-paths.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +# Sourceable path lists for files owned by the Termux release automation. + +set -euo pipefail + +readonly -a TERMUX_RELEASE_WORKFLOW_PATHS=( + .github/workflows/rust-release.yml + .github/workflows/shell-tool-mcp.yml + .github/workflows/termux-release-checkpoint.yml + .github/workflows/termux-release-deploy.yml + .github/workflows/termux-release-promote.yml +) + +readonly -a TERMUX_RELEASE_BRANCH_SCRIPT_PATHS=( + scripts/termux-configure-git.sh + scripts/termux-create-checkpoint-pr.sh + scripts/termux-create-or-update-mirrored-release.sh + scripts/termux-download-release-artifact.sh + scripts/termux-find-release-pr.sh + scripts/termux-read-release-metadata.sh + scripts/termux-release-asset-state.sh + scripts/termux-release-paths.sh + scripts/termux-resolve-release-ref.sh + scripts/termux-validate-gh-env.sh +) + +readonly -a TERMUX_RELEASE_AUTOMATION_PATHS=( + "${TERMUX_RELEASE_WORKFLOW_PATHS[@]}" + "${TERMUX_RELEASE_BRANCH_SCRIPT_PATHS[@]}" +) + +readonly -a TERMUX_CHECKPOINT_RELEASE_ONLY_PATHS=( + "${TERMUX_RELEASE_AUTOMATION_PATHS[@]}" + .github/termux-release.json +) + +termux_path_in_list() { + local candidate="$1" + shift + local listed_path + + for listed_path in "$@"; do + [[ "${candidate}" != "${listed_path}" ]] || return 0 + done + return 1 +} + +termux_is_release_automation_path() { + termux_path_in_list "$1" "${TERMUX_RELEASE_AUTOMATION_PATHS[@]}" +} + +termux_is_checkpoint_release_only_path() { + termux_path_in_list "$1" "${TERMUX_CHECKPOINT_RELEASE_ONLY_PATHS[@]}" +} diff --git a/scripts/termux-resolve-release-ref.sh b/scripts/termux-resolve-release-ref.sh new file mode 100755 index 00000000000..194773d45be --- /dev/null +++ b/scripts/termux-resolve-release-ref.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Resolve the release branch and SHA for deploy/promote workflows. + +set -euo pipefail + +: "${GITHUB_EVENT_NAME:?GITHUB_EVENT_NAME is required}" +: "${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}" + +input_release_branch="${INPUT_RELEASE_BRANCH:-${REQUESTED_RELEASE_BRANCH:-}}" +input_release_sha="${INPUT_RELEASE_SHA:-${REQUESTED_RELEASE_SHA:-}}" + +if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then + if [[ -z "${input_release_branch}" ]]; then + echo "release_branch is required for workflow_dispatch." >&2 + exit 1 + fi + + release_branch="${input_release_branch}" + if [[ -n "${input_release_sha}" ]]; then + git checkout --detach "${input_release_sha}" + release_sha="${input_release_sha}" + else + release_sha="$(git rev-parse HEAD)" + fi +else + : "${GITHUB_REF_NAME:?GITHUB_REF_NAME is required}" + : "${GITHUB_SHA:?GITHUB_SHA is required}" + release_branch="${GITHUB_REF_NAME}" + release_sha="${GITHUB_SHA}" +fi + +{ + echo "branch=${release_branch}" + echo "sha=${release_sha}" +} >> "${GITHUB_OUTPUT}" diff --git a/scripts/termux-validate-gh-env.sh b/scripts/termux-validate-gh-env.sh new file mode 100755 index 00000000000..02581fdebce --- /dev/null +++ b/scripts/termux-validate-gh-env.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# Lightweight post-toolbox check for jobs that rely on authenticated gh calls. + +set -euo pipefail + +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}" +: "${GH_TOKEN:?GH_TOKEN is required}" + +command -v gh +gh auth status --hostname github.com + +printf 'GITHUB_REPOSITORY=%s\n' "${GITHUB_REPOSITORY}" +printf 'REPO=%s\n' "${REPO:-}" +printf 'GH_REPO_URL=%s\n' "${GH_REPO_URL:-}" +printf 'GH_WORKFLOW_URL=%s\n' "${GH_WORKFLOW_URL:-}"