From c743b25a5160deb4d51b4c0accf24956bdb91ba8 Mon Sep 17 00:00:00 2001 From: Allister MacLeod Date: Mon, 22 Jun 2026 13:47:00 -0400 Subject: [PATCH 1/5] Throttle Unity CI license activations to the per-serial seat budget The PR Build Unity matrix fired all 14 jobs (10 tests + 4 device builds) at once against 3 Unity Pro serials of 2 seats each. With demand (14) far above the 6-seat ceiling, a random subset of jobs lost the activation race each run and failed with '404 / found 0 entitlement groups', producing red checks that are pure license contention rather than real defects. Over the last ~4 weeks this hit ~half of all PR Build runs. Shard the Unity work into one job per serial, each capped at max-parallel: 2, with serials assigned so no two jobs share a serial: - testGroupDefault -> UNITY_SERIAL (2021.3.29, 2022.3.7, 6000.0.37) - testGroupBuild -> UNITY_SERIAL_BUILD (6000.2.8, 6000.3.0) - build -> UNITY_SERIAL_DEVOPS (all 4 device platforms) Concurrent activations on any serial can now never exceed its 2 seats. Per-cell check names ('Tests {ver} {mode}', 'Build for {platform}') are unchanged, so branch-protection required checks keep matching. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/buildPR.yml | 187 ++++++++++++++++++++++++++++------ 1 file changed, 156 insertions(+), 31 deletions(-) diff --git a/.github/workflows/buildPR.yml b/.github/workflows/buildPR.yml index 225f12b527..e741c5ee7d 100644 --- a/.github/workflows/buildPR.yml +++ b/.github/workflows/buildPR.yml @@ -179,7 +179,13 @@ jobs: run: dotnet build --configuration Release --no-restore templates/SourceGen/MicroserviceSourceGen.Tests - name: Test run: dotnet test --configuration Release --no-build templates/SourceGen/MicroserviceSourceGen.Tests - testAllModes: + # Unity Pro serials allow 2 concurrent activations each. We own 3 serials + # (UNITY_SERIAL, UNITY_SERIAL_BUILD, UNITY_SERIAL_DEVOPS = 6 seats total), so the + # Unity workload is sharded into one job per serial, each capped at max-parallel: 2. + # Each serial is used by exactly one job, so concurrent activations on a serial can + # never exceed its 2 seats and we stop tripping spurious "0 entitlement groups" + # license-activation failures. Bump max-parallel if a serial's seat count changes. ~Claude + testGroupDefault: timeout-minutes: 30 name: Tests ${{ matrix.unityVersion }} ${{ matrix.testMode }} runs-on: ubuntu-latest @@ -188,6 +194,7 @@ jobs: cancel-in-progress: true strategy: fail-fast: false + max-parallel: 2 matrix: projectPath: - client @@ -198,23 +205,8 @@ jobs: - 2021.3.29f1 - 2022.3.7f1 - 6000.0.37f1 - - 6000.2.8f1 - - 6000.3.0f1 gameCiVersion: - 3.1.0 - include: - - unityVersion: 2022.3.7f1 - unityEmailOverride: UNITY_EMAIL_DEVOPS - unityPasswordOverride: UNITY_PASSWORD_DEVOPS - unitySerialOverride: UNITY_SERIAL_DEVOPS - - unityVersion: 2021.3.29f1 - unityEmailOverride: UNITY_EMAIL_BUILD - unityPasswordOverride: UNITY_PASSWORD_BUILD - unitySerialOverride: UNITY_SERIAL_BUILD - - unityVersion: 6000.0.37f1 - unityEmailOverride: UNITY_EMAIL_BUILD - unityPasswordOverride: UNITY_PASSWORD_BUILD - unitySerialOverride: UNITY_SERIAL_BUILD env: GAME_CI_VERSION: >- ${{ @@ -318,9 +310,148 @@ jobs: LANG: en_US.UTF-8 LC_ALL: en_US.UTF-8 BEAM_UNITY_TEST_CI: "true" - UNITY_EMAIL: ${{ secrets[matrix.unityEmailOverride] || secrets.UNITY_EMAIL }} - UNITY_PASSWORD: ${{ secrets[matrix.unityPasswordOverride] || secrets.UNITY_PASSWORD }} - UNITY_SERIAL: ${{ secrets[matrix.unitySerialOverride] || secrets.UNITY_SERIAL }} + UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} + UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} + UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} + with: + customImage: ${{ env.DOCKER_REGISTRY }}/beamable/editor:ubuntu-${{ matrix.unityVersion }}-base-${{ env.GAME_CI_VERSION }} + customParameters: -warnaserror+ -executeMethod Beamable.Editor.BeamCli.BeamCliUtil.InstallToolFromLocalPackageSource + projectPath: ${{ matrix.projectPath }} + testMode: ${{ matrix.testMode }} + unityVersion: ${{ matrix.unityVersion }} + artifactsPath: ${{ matrix.testMode }}-${{ matrix.unityVersion }}-artifacts + githubToken: ${{ secrets.GITHUB_TOKEN }} + checkName: ${{ matrix.unityVersion }} ${{ matrix.testMode }} Test Results + + # Second Unity-test shard, pinned to the UNITY_SERIAL_BUILD seat (see the comment + # above testGroupDefault for the seat-budget rationale). ~Claude + testGroupBuild: + timeout-minutes: 30 + name: Tests ${{ matrix.unityVersion }} ${{ matrix.testMode }} + runs-on: ubuntu-latest + concurrency: + group: unity-tests-${{ matrix.unityVersion }}-${{ matrix.testMode }}-${{ github.head_ref }} + cancel-in-progress: true + strategy: + fail-fast: false + max-parallel: 2 + matrix: + projectPath: + - client + testMode: + - playmode + - editmode + unityVersion: + - 6000.2.8f1 + - 6000.3.0f1 + gameCiVersion: + - 3.1.0 + env: + GAME_CI_VERSION: >- + ${{ + matrix.unityVersion == '6000.2.8f1' && '3.2.0' || + matrix.unityVersion == '6000.3.0f1' && '3.2.1' || + matrix.gameCiVersion + }} + steps: + - uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.DOCKER_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create Unity Mapped Values + id: unityMap + env: + UNITY_MODE: >- + ${{ fromJson('{ + "playmode": "player", + "editmode": "editor" + }')[matrix.testMode] }} + run: | + echo "UNITY_MODE=$UNITY_MODE" >> "$GITHUB_OUTPUT" + + + - name: Setup .NET Core SDK ${{ matrix.dotnet-version }} + uses: actions/setup-dotnet@v4 + with: + dotnet-version: | + 10.0.x + 9.0.x + 8.0.302 + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.24.1 + cache-dependency-path: ./otel-collector/beamable-collector/go.sum + - name: Build and Install BEAM CLI + working-directory: ./ + run: | + ./setup.sh + SKIP_GENERATION=true ./dev.sh --skip-unreal + env: + PROJECT_DIR_OVERRIDE: ${{ github.workspace }}/NugetBuildDir/ + - name: Check Beam CLI + run: | + pwd + ls -a + beam version + - name: Apply Nuget Version to Unity SDK + run: | + ReleaseSharedCode=true dotnet build ./cli/beamable.common -t:CopyCodeToUnity + + - name: Apply CLI Versions + run: jq --arg nextVersion 0.0.123.1 '.nugetPackageVersion = $nextVersion' ./client/Packages/com.beamable/Runtime/Environment/Resources/versions-default.json > localtmpfile && mv localtmpfile ./client/Packages/com.beamable/Runtime/Environment/Resources/versions-default.json + + - name: Review Unity version-default file + run: | + cat ./client/Packages/com.beamable/Runtime/Environment/Resources/versions-default.json + + - name: Copy the nuget packages into unity + working-directory: ./ + run: | + mkdir -p client/BeamableNugetSource/ + ls BeamableNugetSource + cp BeamableNugetSource/*.nupkg client/BeamableNugetSource/ + + - name: Cache Unity Folders + uses: actions/cache@v3 + with: + path: ${{ matrix.projectPath }}/Library + key: Library-2-${{ matrix.projectPath }}-${{ matrix.unityVersion }} + restore-keys: | + Library-2-${{ matrix.projectPath }}-${{ matrix.unityVersion }} + + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + - uses: game-ci/unity-test-runner@v4 + id: tests + timeout-minutes: 30 + env: + DOTNET_CLI_UI_LANGUAGE: en + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: 1 + LANG: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + BEAM_UNITY_TEST_CI: "true" + UNITY_EMAIL: ${{ secrets.UNITY_EMAIL_BUILD }} + UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD_BUILD }} + UNITY_SERIAL: ${{ secrets.UNITY_SERIAL_BUILD }} with: customImage: ${{ env.DOCKER_REGISTRY }}/beamable/editor:ubuntu-${{ matrix.unityVersion }}-base-${{ env.GAME_CI_VERSION }} customParameters: -warnaserror+ -executeMethod Beamable.Editor.BeamCli.BeamCliUtil.InstallToolFromLocalPackageSource @@ -340,6 +471,9 @@ jobs: cancel-in-progress: true strategy: fail-fast: false + # All device builds share the UNITY_SERIAL_DEVOPS seat (see the comment above + # testGroupDefault). max-parallel: 2 keeps activations within that serial's 2 seats. ~Claude + max-parallel: 2 matrix: targetPlatform: - Android @@ -350,15 +484,6 @@ jobs: - 6000.0.37f1 projectPath: - client - include: - - targetPlatform: Android - unityEmailOverride: UNITY_EMAIL_DEVOPS - unityPasswordOverride: UNITY_PASSWORD_DEVOPS - unitySerialOverride: UNITY_SERIAL_DEVOPS - - targetPlatform: StandaloneOSX - unityEmailOverride: UNITY_EMAIL_BUILD - unityPasswordOverride: UNITY_PASSWORD_BUILD - unitySerialOverride: UNITY_SERIAL_BUILD steps: - uses: actions/checkout@v4 with: @@ -446,9 +571,9 @@ jobs: - uses: game-ci/unity-builder@v4 timeout-minutes: 60 env: - UNITY_EMAIL: ${{ secrets[matrix.unityEmailOverride] || secrets.UNITY_EMAIL }} - UNITY_PASSWORD: ${{ secrets[matrix.unityPasswordOverride] || secrets.UNITY_PASSWORD }} - UNITY_SERIAL: ${{ secrets[matrix.unitySerialOverride] || secrets.UNITY_SERIAL }} + UNITY_EMAIL: ${{ secrets.UNITY_EMAIL_DEVOPS }} + UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD_DEVOPS }} + UNITY_SERIAL: ${{ secrets.UNITY_SERIAL_DEVOPS }} with: customImage: ${{ env.DOCKER_REGISTRY }}/beamable/editor:ubuntu-${{ matrix.unityVersion }}-${{steps.unityMap.outputs.UNITY_PLATFORM}}-3.1.0 targetPlatform: ${{ matrix.targetPlatform }} From 4cd69b7b06e72ba3d75538b66d4062af37f4c3f4 Mon Sep 17 00:00:00 2001 From: Allister MacLeod Date: Tue, 23 Jun 2026 11:37:33 -0400 Subject: [PATCH 2/5] Cap Unity test shards at max-parallel 1 Run 27972570615 showed that capping each per-serial shard at max-parallel 2 still tripped "No valid Unity Editor license found": the license return of a finishing matrix leg overlaps the activation of the next, momentarily demanding a third seat against a 2-seat serial. Dropping to 1 leaves a full seat of headroom for that return/activate transition. Costs ~2x wall clock on the Unity test stage (~30m -> ~60m) at identical runner-minutes and halved peak runner concurrency. Does not address cross-run contention. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/buildPR.yml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/buildPR.yml b/.github/workflows/buildPR.yml index e741c5ee7d..0c68170da9 100644 --- a/.github/workflows/buildPR.yml +++ b/.github/workflows/buildPR.yml @@ -179,12 +179,15 @@ jobs: run: dotnet build --configuration Release --no-restore templates/SourceGen/MicroserviceSourceGen.Tests - name: Test run: dotnet test --configuration Release --no-build templates/SourceGen/MicroserviceSourceGen.Tests - # Unity Pro serials allow 2 concurrent activations each. We own 3 serials - # (UNITY_SERIAL, UNITY_SERIAL_BUILD, UNITY_SERIAL_DEVOPS = 6 seats total), so the - # Unity workload is sharded into one job per serial, each capped at max-parallel: 2. - # Each serial is used by exactly one job, so concurrent activations on a serial can - # never exceed its 2 seats and we stop tripping spurious "0 entitlement groups" - # license-activation failures. Bump max-parallel if a serial's seat count changes. ~Claude + # Unity Pro serials allow 2 concurrent activations each, and we own 3 serials + # (UNITY_SERIAL, UNITY_SERIAL_BUILD, UNITY_SERIAL_DEVOPS), so the Unity workload is + # sharded into one job per serial. Each shard runs max-parallel: 1. Although a serial + # has 2 seats, the license return of a finishing leg overlaps the activation of the + # next, momentarily demanding a third seat and tripping spurious "No valid Unity Editor + # license found" failures (observed on run 27972570615). Capping at 1 leaves a full + # seat of headroom for that return/activate transition. This does not guard against + # cross-run contention (a second concurrent run on the same serial); raise toward the + # seat count only behind a license server that gives blocking seat acquisition. ~Claude testGroupDefault: timeout-minutes: 30 name: Tests ${{ matrix.unityVersion }} ${{ matrix.testMode }} @@ -194,7 +197,7 @@ jobs: cancel-in-progress: true strategy: fail-fast: false - max-parallel: 2 + max-parallel: 1 matrix: projectPath: - client @@ -334,7 +337,7 @@ jobs: cancel-in-progress: true strategy: fail-fast: false - max-parallel: 2 + max-parallel: 1 matrix: projectPath: - client @@ -472,8 +475,9 @@ jobs: strategy: fail-fast: false # All device builds share the UNITY_SERIAL_DEVOPS seat (see the comment above - # testGroupDefault). max-parallel: 2 keeps activations within that serial's 2 seats. ~Claude - max-parallel: 2 + # testGroupDefault). max-parallel: 1 leaves a seat of headroom for the license + # return/activate transition. ~Claude + max-parallel: 1 matrix: targetPlatform: - Android From a861bb6daf6cf8dd2aa47ecfdf7f8097ed2b6989 Mon Sep 17 00:00:00 2001 From: Allister MacLeod Date: Tue, 23 Jun 2026 14:29:35 -0400 Subject: [PATCH 3/5] Annotate Unity license contention on failed test shards When a Unity test shard fails without producing any result XML, the editor exited before tests ran -- the signature of a license/seat activation failure. Emit a GitHub ::error annotation naming license contention so the failed job is self-explanatory at a glance instead of presenting only a generic "docker failed with exit code 1". Uses artifact presence rather than parsing the editor stdout, which a later step cannot read back; all Unity jobs run on uniform ubuntu-latest hosts, so the signal is consistent. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/buildPR.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.github/workflows/buildPR.yml b/.github/workflows/buildPR.yml index 0c68170da9..6913926e7d 100644 --- a/.github/workflows/buildPR.yml +++ b/.github/workflows/buildPR.yml @@ -326,6 +326,24 @@ jobs: githubToken: ${{ secrets.GITHUB_TOKEN }} checkName: ${{ matrix.unityVersion }} ${{ matrix.testMode }} Test Results + # A license/seat activation failure aborts before any tests run, so it produces + # no result XML. If the shard failed and no XML exists, surface a plain-language + # annotation naming license contention so the red build is self-explanatory and + # nobody has to scroll the editor log to find "No valid Unity Editor license + # found". (Artifact presence is used rather than parsing the editor stdout, which + # a later step cannot read back.) ~Claude + - name: Annotate Unity license contention + if: failure() + shell: bash + env: + ARTIFACTS: ${{ matrix.testMode }}-${{ matrix.unityVersion }}-artifacts + run: | + if find "$ARTIFACTS" -name '*.xml' 2>/dev/null | grep -q .; then + echo "Test result XML present: this is a genuine test failure, not license contention." + else + echo "::error title=Unity license contention::${{ matrix.unityVersion }} ${{ matrix.testMode }} produced no test results -- the editor exited before tests ran, the signature of a Unity license/seat activation failure (serial contention), not a test failure." + fi + # Second Unity-test shard, pinned to the UNITY_SERIAL_BUILD seat (see the comment # above testGroupDefault for the seat-budget rationale). ~Claude testGroupBuild: @@ -465,6 +483,19 @@ jobs: githubToken: ${{ secrets.GITHUB_TOKEN }} checkName: ${{ matrix.unityVersion }} ${{ matrix.testMode }} Test Results + # See the annotation rationale above testGroupDefault's matching step. ~Claude + - name: Annotate Unity license contention + if: failure() + shell: bash + env: + ARTIFACTS: ${{ matrix.testMode }}-${{ matrix.unityVersion }}-artifacts + run: | + if find "$ARTIFACTS" -name '*.xml' 2>/dev/null | grep -q .; then + echo "Test result XML present: this is a genuine test failure, not license contention." + else + echo "::error title=Unity license contention::${{ matrix.unityVersion }} ${{ matrix.testMode }} produced no test results -- the editor exited before tests ran, the signature of a Unity license/seat activation failure (serial contention), not a test failure." + fi + build: name: Build for ${{ matrix.targetPlatform }} runs-on: ubuntu-latest From 21d5bf55f0b887866f9459ce32711487adcdffc4 Mon Sep 17 00:00:00 2001 From: Allister MacLeod Date: Tue, 23 Jun 2026 14:42:18 -0400 Subject: [PATCH 4/5] Add Unity license-contention problem matcher (mid-run annotations) Register a problem matcher before each Unity test shard's game-ci step so the literal license-contention lines ("No valid Unity Editor license found", "Serial number unavailable for ULF return") surface as error annotations live, during the run. This is the belt to the XML-check step's suspenders: the matcher gives an immediate mid-run tidbit quoting Unity verbatim, while the post-failure XML check classifies whether the job died before tests ran. Problem matchers are a native runner feature (no dependency). Whether they scan container-action stdout is unconfirmed and should be verified on a real run. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/matchers/unity-license.json | 18 ++++++++++++++++++ .github/workflows/buildPR.yml | 8 ++++++++ 2 files changed, 26 insertions(+) create mode 100644 .github/matchers/unity-license.json diff --git a/.github/matchers/unity-license.json b/.github/matchers/unity-license.json new file mode 100644 index 0000000000..7f584b3f4a --- /dev/null +++ b/.github/matchers/unity-license.json @@ -0,0 +1,18 @@ +{ + "problemMatcher": [ + { + "owner": "unity-license-no-valid-license", + "severity": "error", + "pattern": [ + { "regexp": "^.*(No valid Unity Editor license found.*)$", "message": 1 } + ] + }, + { + "owner": "unity-license-ulf-return-failed", + "severity": "error", + "pattern": [ + { "regexp": "^.*(Serial number unavailable for ULF return.*)$", "message": 1 } + ] + } + ] +} diff --git a/.github/workflows/buildPR.yml b/.github/workflows/buildPR.yml index 6913926e7d..5a1901b7e0 100644 --- a/.github/workflows/buildPR.yml +++ b/.github/workflows/buildPR.yml @@ -304,6 +304,11 @@ jobs: large-packages: true docker-images: true swap-storage: true + # Surface the literal Unity license-contention lines as annotations live, mid-run. + # Complements the post-failure XML check below, which controls the job's outcome. + # Container-action stdout scope is unconfirmed; verify on a real run. ~Claude + - name: Register Unity license problem matcher + run: echo "::add-matcher::.github/matchers/unity-license.json" - uses: game-ci/unity-test-runner@v4 id: tests timeout-minutes: 30 @@ -461,6 +466,9 @@ jobs: large-packages: true docker-images: true swap-storage: true + # See the registration rationale above testGroupDefault's matching step. ~Claude + - name: Register Unity license problem matcher + run: echo "::add-matcher::.github/matchers/unity-license.json" - uses: game-ci/unity-test-runner@v4 id: tests timeout-minutes: 30 From 018ed66dbc1b328b7a652ff95ada8197f5afb445 Mon Sep 17 00:00:00 2001 From: Allister MacLeod Date: Tue, 23 Jun 2026 14:18:32 -0400 Subject: [PATCH 5/5] Retry Unity tests on license-contention failures with jittered backoff Add a composite action wrapping game-ci/unity-test-runner that retries up to 3 times with exponential backoff plus jitter, but only when a run produced no result XML (the signature of a license/seat activation failure, which aborts before tests execute). Runs that produced results -- pass or fail -- surface immediately, so genuine test failures are never masked or slowed. Wired into the two unity-test-runner shards (testGroupDefault, testGroupBuild); device builds use unity-builder and are out of scope. Shard timeouts raised 30 -> 75 to cover the extra attempts. Stacked on the max-parallel:1 change (#4702), which targets the intra-run transition race; this targets residual cross-run contention. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../actions/unity-test-with-retry/action.yml | 182 ++++++++++++++++++ .github/workflows/buildPR.yml | 56 +++--- 2 files changed, 204 insertions(+), 34 deletions(-) create mode 100644 .github/actions/unity-test-with-retry/action.yml diff --git a/.github/actions/unity-test-with-retry/action.yml b/.github/actions/unity-test-with-retry/action.yml new file mode 100644 index 0000000000..3c86c305d8 --- /dev/null +++ b/.github/actions/unity-test-with-retry/action.yml @@ -0,0 +1,182 @@ +# Wrap game-ci/unity-test-runner with patient, jittered retries that fire ONLY on +# license-activation failures. A license/seat failure aborts before any test results +# are written, so "no result XML was produced" is used as the signal to retry; a run +# that produced result XML (pass OR fail) is a genuine outcome and surfaces immediately +# without masking. Up to 3 attempts with exponential backoff plus jitter. ~Claude +name: Unity test runner with license-contention retry +description: Run game-ci/unity-test-runner, retrying only on license-activation failures with jittered backoff. + +inputs: + unityVersion: + required: true + description: Unity editor version. + testMode: + required: true + description: playmode or editmode. + projectPath: + required: true + description: Path to the Unity project. + customImage: + required: true + description: Editor container image. + customParameters: + required: false + default: "" + description: Extra parameters forwarded to the editor. + artifactsPath: + required: true + description: Where unity-test-runner writes result XML. Used to classify failures. + checkName: + required: true + description: Name for the published Test Results check. + unityEmail: + required: true + description: Unity account email (passed in; composite actions cannot read secrets directly). + unityPassword: + required: true + description: Unity account password. + unitySerial: + required: true + description: Unity serial whose seats this shard activates. + githubToken: + required: true + description: Token for publishing the Test Results check. + backoffSeconds: + required: false + default: "30" + description: Base backoff; attempt N waits backoff*2^(N-1) plus jitter. + jitterSeconds: + required: false + default: "45" + description: Upper bound of the random jitter added to each backoff. + +runs: + using: composite + steps: + - name: Unity tests (attempt 1) + id: a1 + continue-on-error: true + uses: game-ci/unity-test-runner@v4 + env: + DOTNET_CLI_UI_LANGUAGE: en + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: 1 + LANG: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + BEAM_UNITY_TEST_CI: "true" + UNITY_EMAIL: ${{ inputs.unityEmail }} + UNITY_PASSWORD: ${{ inputs.unityPassword }} + UNITY_SERIAL: ${{ inputs.unitySerial }} + with: + customImage: ${{ inputs.customImage }} + customParameters: ${{ inputs.customParameters }} + projectPath: ${{ inputs.projectPath }} + testMode: ${{ inputs.testMode }} + unityVersion: ${{ inputs.unityVersion }} + artifactsPath: ${{ inputs.artifactsPath }} + githubToken: ${{ inputs.githubToken }} + checkName: ${{ inputs.checkName }} + + - name: Classify attempt 1 and back off + id: g1 + if: steps.a1.outcome == 'failure' + shell: bash + env: + ARTIFACTS: ${{ inputs.artifactsPath }} + BACKOFF: ${{ inputs.backoffSeconds }} + JITTER: ${{ inputs.jitterSeconds }} + ATTEMPT: "1" + run: | + set -uo pipefail + if find "$ARTIFACTS" -name '*.xml' 2>/dev/null | grep -q .; then + echo "Result XML present -> genuine test outcome; not retrying." + echo "retry=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "No result XML -> license/activation failure suspected; will retry." + rm -rf "$ARTIFACTS" 2>/dev/null || true + delay=$(( BACKOFF * (2 ** (ATTEMPT - 1)) + (RANDOM % (JITTER + 1)) )) + echo "Backing off ${delay}s (jittered) before attempt $((ATTEMPT + 1))." + sleep "$delay" + echo "retry=true" >> "$GITHUB_OUTPUT" + + - name: Unity tests (attempt 2) + id: a2 + if: steps.a1.outcome == 'failure' && steps.g1.outputs.retry == 'true' + continue-on-error: true + uses: game-ci/unity-test-runner@v4 + env: + DOTNET_CLI_UI_LANGUAGE: en + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: 1 + LANG: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + BEAM_UNITY_TEST_CI: "true" + UNITY_EMAIL: ${{ inputs.unityEmail }} + UNITY_PASSWORD: ${{ inputs.unityPassword }} + UNITY_SERIAL: ${{ inputs.unitySerial }} + with: + customImage: ${{ inputs.customImage }} + customParameters: ${{ inputs.customParameters }} + projectPath: ${{ inputs.projectPath }} + testMode: ${{ inputs.testMode }} + unityVersion: ${{ inputs.unityVersion }} + artifactsPath: ${{ inputs.artifactsPath }} + githubToken: ${{ inputs.githubToken }} + checkName: ${{ inputs.checkName }} + + - name: Classify attempt 2 and back off + id: g2 + if: steps.a2.outcome == 'failure' && steps.g1.outputs.retry == 'true' + shell: bash + env: + ARTIFACTS: ${{ inputs.artifactsPath }} + BACKOFF: ${{ inputs.backoffSeconds }} + JITTER: ${{ inputs.jitterSeconds }} + ATTEMPT: "2" + run: | + set -uo pipefail + if find "$ARTIFACTS" -name '*.xml' 2>/dev/null | grep -q .; then + echo "Result XML present -> genuine test outcome; not retrying." + echo "retry=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "No result XML -> license/activation failure suspected; will retry." + rm -rf "$ARTIFACTS" 2>/dev/null || true + delay=$(( BACKOFF * (2 ** (ATTEMPT - 1)) + (RANDOM % (JITTER + 1)) )) + echo "Backing off ${delay}s (jittered) before attempt $((ATTEMPT + 1))." + sleep "$delay" + echo "retry=true" >> "$GITHUB_OUTPUT" + + # Final attempt: NOT continue-on-error, so an exhausted-retry failure fails the job. + - name: Unity tests (attempt 3, final) + id: a3 + if: steps.a2.outcome == 'failure' && steps.g2.outputs.retry == 'true' + uses: game-ci/unity-test-runner@v4 + env: + DOTNET_CLI_UI_LANGUAGE: en + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: 1 + LANG: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + BEAM_UNITY_TEST_CI: "true" + UNITY_EMAIL: ${{ inputs.unityEmail }} + UNITY_PASSWORD: ${{ inputs.unityPassword }} + UNITY_SERIAL: ${{ inputs.unitySerial }} + with: + customImage: ${{ inputs.customImage }} + customParameters: ${{ inputs.customParameters }} + projectPath: ${{ inputs.projectPath }} + testMode: ${{ inputs.testMode }} + unityVersion: ${{ inputs.unityVersion }} + artifactsPath: ${{ inputs.artifactsPath }} + githubToken: ${{ inputs.githubToken }} + checkName: ${{ inputs.checkName }} + + # A continue-on-error attempt that failed with produced results is a genuine + # failure the loop deliberately did not retry; re-surface it as a job failure. + - name: Surface genuine test failure + if: >- + (steps.a1.outcome == 'failure' && steps.g1.outputs.retry == 'false') || + (steps.a2.outcome == 'failure' && steps.g2.outputs.retry == 'false') + shell: bash + run: | + echo "::error::Unity tests failed with produced results (genuine failure, not a license flake)." + exit 1 diff --git a/.github/workflows/buildPR.yml b/.github/workflows/buildPR.yml index 5a1901b7e0..c5b594c014 100644 --- a/.github/workflows/buildPR.yml +++ b/.github/workflows/buildPR.yml @@ -189,7 +189,8 @@ jobs: # cross-run contention (a second concurrent run on the same serial); raise toward the # seat count only behind a license server that gives blocking seat acquisition. ~Claude testGroupDefault: - timeout-minutes: 30 + # Raised from 30 to cover up to 3 license-retry attempts plus jittered backoff. ~Claude + timeout-minutes: 75 name: Tests ${{ matrix.unityVersion }} ${{ matrix.testMode }} runs-on: ubuntu-latest concurrency: @@ -309,27 +310,20 @@ jobs: # Container-action stdout scope is unconfirmed; verify on a real run. ~Claude - name: Register Unity license problem matcher run: echo "::add-matcher::.github/matchers/unity-license.json" - - uses: game-ci/unity-test-runner@v4 - id: tests - timeout-minutes: 30 - env: - DOTNET_CLI_UI_LANGUAGE: en - DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: 1 - LANG: en_US.UTF-8 - LC_ALL: en_US.UTF-8 - BEAM_UNITY_TEST_CI: "true" - UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} - UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} - UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} + - name: Unity tests (license-contention retry) + uses: ./.github/actions/unity-test-with-retry with: + unityVersion: ${{ matrix.unityVersion }} + testMode: ${{ matrix.testMode }} + projectPath: ${{ matrix.projectPath }} customImage: ${{ env.DOCKER_REGISTRY }}/beamable/editor:ubuntu-${{ matrix.unityVersion }}-base-${{ env.GAME_CI_VERSION }} customParameters: -warnaserror+ -executeMethod Beamable.Editor.BeamCli.BeamCliUtil.InstallToolFromLocalPackageSource - projectPath: ${{ matrix.projectPath }} - testMode: ${{ matrix.testMode }} - unityVersion: ${{ matrix.unityVersion }} artifactsPath: ${{ matrix.testMode }}-${{ matrix.unityVersion }}-artifacts - githubToken: ${{ secrets.GITHUB_TOKEN }} checkName: ${{ matrix.unityVersion }} ${{ matrix.testMode }} Test Results + unityEmail: ${{ secrets.UNITY_EMAIL }} + unityPassword: ${{ secrets.UNITY_PASSWORD }} + unitySerial: ${{ secrets.UNITY_SERIAL }} + githubToken: ${{ secrets.GITHUB_TOKEN }} # A license/seat activation failure aborts before any tests run, so it produces # no result XML. If the shard failed and no XML exists, surface a plain-language @@ -352,7 +346,8 @@ jobs: # Second Unity-test shard, pinned to the UNITY_SERIAL_BUILD seat (see the comment # above testGroupDefault for the seat-budget rationale). ~Claude testGroupBuild: - timeout-minutes: 30 + # Raised from 30 to cover up to 3 license-retry attempts plus jittered backoff. ~Claude + timeout-minutes: 75 name: Tests ${{ matrix.unityVersion }} ${{ matrix.testMode }} runs-on: ubuntu-latest concurrency: @@ -469,27 +464,20 @@ jobs: # See the registration rationale above testGroupDefault's matching step. ~Claude - name: Register Unity license problem matcher run: echo "::add-matcher::.github/matchers/unity-license.json" - - uses: game-ci/unity-test-runner@v4 - id: tests - timeout-minutes: 30 - env: - DOTNET_CLI_UI_LANGUAGE: en - DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: 1 - LANG: en_US.UTF-8 - LC_ALL: en_US.UTF-8 - BEAM_UNITY_TEST_CI: "true" - UNITY_EMAIL: ${{ secrets.UNITY_EMAIL_BUILD }} - UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD_BUILD }} - UNITY_SERIAL: ${{ secrets.UNITY_SERIAL_BUILD }} + - name: Unity tests (license-contention retry) + uses: ./.github/actions/unity-test-with-retry with: + unityVersion: ${{ matrix.unityVersion }} + testMode: ${{ matrix.testMode }} + projectPath: ${{ matrix.projectPath }} customImage: ${{ env.DOCKER_REGISTRY }}/beamable/editor:ubuntu-${{ matrix.unityVersion }}-base-${{ env.GAME_CI_VERSION }} customParameters: -warnaserror+ -executeMethod Beamable.Editor.BeamCli.BeamCliUtil.InstallToolFromLocalPackageSource - projectPath: ${{ matrix.projectPath }} - testMode: ${{ matrix.testMode }} - unityVersion: ${{ matrix.unityVersion }} artifactsPath: ${{ matrix.testMode }}-${{ matrix.unityVersion }}-artifacts - githubToken: ${{ secrets.GITHUB_TOKEN }} checkName: ${{ matrix.unityVersion }} ${{ matrix.testMode }} Test Results + unityEmail: ${{ secrets.UNITY_EMAIL_BUILD }} + unityPassword: ${{ secrets.UNITY_PASSWORD_BUILD }} + unitySerial: ${{ secrets.UNITY_SERIAL_BUILD }} + githubToken: ${{ secrets.GITHUB_TOKEN }} # See the annotation rationale above testGroupDefault's matching step. ~Claude - name: Annotate Unity license contention