diff --git a/.github/actions/unity-test-with-retry/action.yml b/.github/actions/unity-test-with-retry/action.yml new file mode 100644 index 0000000000..3c86c305d8 --- /dev/null +++ b/.github/actions/unity-test-with-retry/action.yml @@ -0,0 +1,182 @@ +# Wrap game-ci/unity-test-runner with patient, jittered retries that fire ONLY on +# license-activation failures. A license/seat failure aborts before any test results +# are written, so "no result XML was produced" is used as the signal to retry; a run +# that produced result XML (pass OR fail) is a genuine outcome and surfaces immediately +# without masking. Up to 3 attempts with exponential backoff plus jitter. ~Claude +name: Unity test runner with license-contention retry +description: Run game-ci/unity-test-runner, retrying only on license-activation failures with jittered backoff. + +inputs: + unityVersion: + required: true + description: Unity editor version. + testMode: + required: true + description: playmode or editmode. + projectPath: + required: true + description: Path to the Unity project. + customImage: + required: true + description: Editor container image. + customParameters: + required: false + default: "" + description: Extra parameters forwarded to the editor. + artifactsPath: + required: true + description: Where unity-test-runner writes result XML. Used to classify failures. + checkName: + required: true + description: Name for the published Test Results check. + unityEmail: + required: true + description: Unity account email (passed in; composite actions cannot read secrets directly). + unityPassword: + required: true + description: Unity account password. + unitySerial: + required: true + description: Unity serial whose seats this shard activates. + githubToken: + required: true + description: Token for publishing the Test Results check. + backoffSeconds: + required: false + default: "30" + description: Base backoff; attempt N waits backoff*2^(N-1) plus jitter. + jitterSeconds: + required: false + default: "45" + description: Upper bound of the random jitter added to each backoff. + +runs: + using: composite + steps: + - name: Unity tests (attempt 1) + id: a1 + continue-on-error: true + uses: game-ci/unity-test-runner@v4 + env: + DOTNET_CLI_UI_LANGUAGE: en + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: 1 + LANG: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + BEAM_UNITY_TEST_CI: "true" + UNITY_EMAIL: ${{ inputs.unityEmail }} + UNITY_PASSWORD: ${{ inputs.unityPassword }} + UNITY_SERIAL: ${{ inputs.unitySerial }} + with: + customImage: ${{ inputs.customImage }} + customParameters: ${{ inputs.customParameters }} + projectPath: ${{ inputs.projectPath }} + testMode: ${{ inputs.testMode }} + unityVersion: ${{ inputs.unityVersion }} + artifactsPath: ${{ inputs.artifactsPath }} + githubToken: ${{ inputs.githubToken }} + checkName: ${{ inputs.checkName }} + + - name: Classify attempt 1 and back off + id: g1 + if: steps.a1.outcome == 'failure' + shell: bash + env: + ARTIFACTS: ${{ inputs.artifactsPath }} + BACKOFF: ${{ inputs.backoffSeconds }} + JITTER: ${{ inputs.jitterSeconds }} + ATTEMPT: "1" + run: | + set -uo pipefail + if find "$ARTIFACTS" -name '*.xml' 2>/dev/null | grep -q .; then + echo "Result XML present -> genuine test outcome; not retrying." + echo "retry=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "No result XML -> license/activation failure suspected; will retry." + rm -rf "$ARTIFACTS" 2>/dev/null || true + delay=$(( BACKOFF * (2 ** (ATTEMPT - 1)) + (RANDOM % (JITTER + 1)) )) + echo "Backing off ${delay}s (jittered) before attempt $((ATTEMPT + 1))." + sleep "$delay" + echo "retry=true" >> "$GITHUB_OUTPUT" + + - name: Unity tests (attempt 2) + id: a2 + if: steps.a1.outcome == 'failure' && steps.g1.outputs.retry == 'true' + continue-on-error: true + uses: game-ci/unity-test-runner@v4 + env: + DOTNET_CLI_UI_LANGUAGE: en + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: 1 + LANG: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + BEAM_UNITY_TEST_CI: "true" + UNITY_EMAIL: ${{ inputs.unityEmail }} + UNITY_PASSWORD: ${{ inputs.unityPassword }} + UNITY_SERIAL: ${{ inputs.unitySerial }} + with: + customImage: ${{ inputs.customImage }} + customParameters: ${{ inputs.customParameters }} + projectPath: ${{ inputs.projectPath }} + testMode: ${{ inputs.testMode }} + unityVersion: ${{ inputs.unityVersion }} + artifactsPath: ${{ inputs.artifactsPath }} + githubToken: ${{ inputs.githubToken }} + checkName: ${{ inputs.checkName }} + + - name: Classify attempt 2 and back off + id: g2 + if: steps.a2.outcome == 'failure' && steps.g1.outputs.retry == 'true' + shell: bash + env: + ARTIFACTS: ${{ inputs.artifactsPath }} + BACKOFF: ${{ inputs.backoffSeconds }} + JITTER: ${{ inputs.jitterSeconds }} + ATTEMPT: "2" + run: | + set -uo pipefail + if find "$ARTIFACTS" -name '*.xml' 2>/dev/null | grep -q .; then + echo "Result XML present -> genuine test outcome; not retrying." + echo "retry=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "No result XML -> license/activation failure suspected; will retry." + rm -rf "$ARTIFACTS" 2>/dev/null || true + delay=$(( BACKOFF * (2 ** (ATTEMPT - 1)) + (RANDOM % (JITTER + 1)) )) + echo "Backing off ${delay}s (jittered) before attempt $((ATTEMPT + 1))." + sleep "$delay" + echo "retry=true" >> "$GITHUB_OUTPUT" + + # Final attempt: NOT continue-on-error, so an exhausted-retry failure fails the job. + - name: Unity tests (attempt 3, final) + id: a3 + if: steps.a2.outcome == 'failure' && steps.g2.outputs.retry == 'true' + uses: game-ci/unity-test-runner@v4 + env: + DOTNET_CLI_UI_LANGUAGE: en + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: 1 + LANG: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + BEAM_UNITY_TEST_CI: "true" + UNITY_EMAIL: ${{ inputs.unityEmail }} + UNITY_PASSWORD: ${{ inputs.unityPassword }} + UNITY_SERIAL: ${{ inputs.unitySerial }} + with: + customImage: ${{ inputs.customImage }} + customParameters: ${{ inputs.customParameters }} + projectPath: ${{ inputs.projectPath }} + testMode: ${{ inputs.testMode }} + unityVersion: ${{ inputs.unityVersion }} + artifactsPath: ${{ inputs.artifactsPath }} + githubToken: ${{ inputs.githubToken }} + checkName: ${{ inputs.checkName }} + + # A continue-on-error attempt that failed with produced results is a genuine + # failure the loop deliberately did not retry; re-surface it as a job failure. + - name: Surface genuine test failure + if: >- + (steps.a1.outcome == 'failure' && steps.g1.outputs.retry == 'false') || + (steps.a2.outcome == 'failure' && steps.g2.outputs.retry == 'false') + shell: bash + run: | + echo "::error::Unity tests failed with produced results (genuine failure, not a license flake)." + exit 1 diff --git a/.github/matchers/unity-license.json b/.github/matchers/unity-license.json new file mode 100644 index 0000000000..7f584b3f4a --- /dev/null +++ b/.github/matchers/unity-license.json @@ -0,0 +1,18 @@ +{ + "problemMatcher": [ + { + "owner": "unity-license-no-valid-license", + "severity": "error", + "pattern": [ + { "regexp": "^.*(No valid Unity Editor license found.*)$", "message": 1 } + ] + }, + { + "owner": "unity-license-ulf-return-failed", + "severity": "error", + "pattern": [ + { "regexp": "^.*(Serial number unavailable for ULF return.*)$", "message": 1 } + ] + } + ] +} diff --git a/.github/workflows/buildPR.yml b/.github/workflows/buildPR.yml index 225f12b527..c5b594c014 100644 --- a/.github/workflows/buildPR.yml +++ b/.github/workflows/buildPR.yml @@ -179,8 +179,18 @@ jobs: run: dotnet build --configuration Release --no-restore templates/SourceGen/MicroserviceSourceGen.Tests - name: Test run: dotnet test --configuration Release --no-build templates/SourceGen/MicroserviceSourceGen.Tests - testAllModes: - timeout-minutes: 30 + # Unity Pro serials allow 2 concurrent activations each, and we own 3 serials + # (UNITY_SERIAL, UNITY_SERIAL_BUILD, UNITY_SERIAL_DEVOPS), so the Unity workload is + # sharded into one job per serial. Each shard runs max-parallel: 1. Although a serial + # has 2 seats, the license return of a finishing leg overlaps the activation of the + # next, momentarily demanding a third seat and tripping spurious "No valid Unity Editor + # license found" failures (observed on run 27972570615). Capping at 1 leaves a full + # seat of headroom for that return/activate transition. This does not guard against + # cross-run contention (a second concurrent run on the same serial); raise toward the + # seat count only behind a license server that gives blocking seat acquisition. ~Claude + testGroupDefault: + # Raised from 30 to cover up to 3 license-retry attempts plus jittered backoff. ~Claude + timeout-minutes: 75 name: Tests ${{ matrix.unityVersion }} ${{ matrix.testMode }} runs-on: ubuntu-latest concurrency: @@ -188,6 +198,7 @@ jobs: cancel-in-progress: true strategy: fail-fast: false + max-parallel: 1 matrix: projectPath: - client @@ -198,23 +209,8 @@ jobs: - 2021.3.29f1 - 2022.3.7f1 - 6000.0.37f1 - - 6000.2.8f1 - - 6000.3.0f1 gameCiVersion: - 3.1.0 - include: - - unityVersion: 2022.3.7f1 - unityEmailOverride: UNITY_EMAIL_DEVOPS - unityPasswordOverride: UNITY_PASSWORD_DEVOPS - unitySerialOverride: UNITY_SERIAL_DEVOPS - - unityVersion: 2021.3.29f1 - unityEmailOverride: UNITY_EMAIL_BUILD - unityPasswordOverride: UNITY_PASSWORD_BUILD - unitySerialOverride: UNITY_SERIAL_BUILD - - unityVersion: 6000.0.37f1 - unityEmailOverride: UNITY_EMAIL_BUILD - unityPasswordOverride: UNITY_PASSWORD_BUILD - unitySerialOverride: UNITY_SERIAL_BUILD env: GAME_CI_VERSION: >- ${{ @@ -309,27 +305,192 @@ jobs: large-packages: true docker-images: true swap-storage: true - - uses: game-ci/unity-test-runner@v4 - id: tests - timeout-minutes: 30 - env: - DOTNET_CLI_UI_LANGUAGE: en - DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: 1 - LANG: en_US.UTF-8 - LC_ALL: en_US.UTF-8 - BEAM_UNITY_TEST_CI: "true" - UNITY_EMAIL: ${{ secrets[matrix.unityEmailOverride] || secrets.UNITY_EMAIL }} - UNITY_PASSWORD: ${{ secrets[matrix.unityPasswordOverride] || secrets.UNITY_PASSWORD }} - UNITY_SERIAL: ${{ secrets[matrix.unitySerialOverride] || secrets.UNITY_SERIAL }} + # Surface the literal Unity license-contention lines as annotations live, mid-run. + # Complements the post-failure XML check below, which controls the job's outcome. + # Container-action stdout scope is unconfirmed; verify on a real run. ~Claude + - name: Register Unity license problem matcher + run: echo "::add-matcher::.github/matchers/unity-license.json" + - name: Unity tests (license-contention retry) + uses: ./.github/actions/unity-test-with-retry with: + unityVersion: ${{ matrix.unityVersion }} + testMode: ${{ matrix.testMode }} + projectPath: ${{ matrix.projectPath }} customImage: ${{ env.DOCKER_REGISTRY }}/beamable/editor:ubuntu-${{ matrix.unityVersion }}-base-${{ env.GAME_CI_VERSION }} customParameters: -warnaserror+ -executeMethod Beamable.Editor.BeamCli.BeamCliUtil.InstallToolFromLocalPackageSource - projectPath: ${{ matrix.projectPath }} - testMode: ${{ matrix.testMode }} - unityVersion: ${{ matrix.unityVersion }} artifactsPath: ${{ matrix.testMode }}-${{ matrix.unityVersion }}-artifacts + checkName: ${{ matrix.unityVersion }} ${{ matrix.testMode }} Test Results + unityEmail: ${{ secrets.UNITY_EMAIL }} + unityPassword: ${{ secrets.UNITY_PASSWORD }} + unitySerial: ${{ secrets.UNITY_SERIAL }} githubToken: ${{ secrets.GITHUB_TOKEN }} + + # A license/seat activation failure aborts before any tests run, so it produces + # no result XML. If the shard failed and no XML exists, surface a plain-language + # annotation naming license contention so the red build is self-explanatory and + # nobody has to scroll the editor log to find "No valid Unity Editor license + # found". (Artifact presence is used rather than parsing the editor stdout, which + # a later step cannot read back.) ~Claude + - name: Annotate Unity license contention + if: failure() + shell: bash + env: + ARTIFACTS: ${{ matrix.testMode }}-${{ matrix.unityVersion }}-artifacts + run: | + if find "$ARTIFACTS" -name '*.xml' 2>/dev/null | grep -q .; then + echo "Test result XML present: this is a genuine test failure, not license contention." + else + echo "::error title=Unity license contention::${{ matrix.unityVersion }} ${{ matrix.testMode }} produced no test results -- the editor exited before tests ran, the signature of a Unity license/seat activation failure (serial contention), not a test failure." + fi + + # Second Unity-test shard, pinned to the UNITY_SERIAL_BUILD seat (see the comment + # above testGroupDefault for the seat-budget rationale). ~Claude + testGroupBuild: + # Raised from 30 to cover up to 3 license-retry attempts plus jittered backoff. ~Claude + timeout-minutes: 75 + name: Tests ${{ matrix.unityVersion }} ${{ matrix.testMode }} + runs-on: ubuntu-latest + concurrency: + group: unity-tests-${{ matrix.unityVersion }}-${{ matrix.testMode }}-${{ github.head_ref }} + cancel-in-progress: true + strategy: + fail-fast: false + max-parallel: 1 + matrix: + projectPath: + - client + testMode: + - playmode + - editmode + unityVersion: + - 6000.2.8f1 + - 6000.3.0f1 + gameCiVersion: + - 3.1.0 + env: + GAME_CI_VERSION: >- + ${{ + matrix.unityVersion == '6000.2.8f1' && '3.2.0' || + matrix.unityVersion == '6000.3.0f1' && '3.2.1' || + matrix.gameCiVersion + }} + steps: + - uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.DOCKER_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create Unity Mapped Values + id: unityMap + env: + UNITY_MODE: >- + ${{ fromJson('{ + "playmode": "player", + "editmode": "editor" + }')[matrix.testMode] }} + run: | + echo "UNITY_MODE=$UNITY_MODE" >> "$GITHUB_OUTPUT" + + + - name: Setup .NET Core SDK ${{ matrix.dotnet-version }} + uses: actions/setup-dotnet@v4 + with: + dotnet-version: | + 10.0.x + 9.0.x + 8.0.302 + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.24.1 + cache-dependency-path: ./otel-collector/beamable-collector/go.sum + - name: Build and Install BEAM CLI + working-directory: ./ + run: | + ./setup.sh + SKIP_GENERATION=true ./dev.sh --skip-unreal + env: + PROJECT_DIR_OVERRIDE: ${{ github.workspace }}/NugetBuildDir/ + - name: Check Beam CLI + run: | + pwd + ls -a + beam version + - name: Apply Nuget Version to Unity SDK + run: | + ReleaseSharedCode=true dotnet build ./cli/beamable.common -t:CopyCodeToUnity + + - name: Apply CLI Versions + run: jq --arg nextVersion 0.0.123.1 '.nugetPackageVersion = $nextVersion' ./client/Packages/com.beamable/Runtime/Environment/Resources/versions-default.json > localtmpfile && mv localtmpfile ./client/Packages/com.beamable/Runtime/Environment/Resources/versions-default.json + + - name: Review Unity version-default file + run: | + cat ./client/Packages/com.beamable/Runtime/Environment/Resources/versions-default.json + + - name: Copy the nuget packages into unity + working-directory: ./ + run: | + mkdir -p client/BeamableNugetSource/ + ls BeamableNugetSource + cp BeamableNugetSource/*.nupkg client/BeamableNugetSource/ + + - name: Cache Unity Folders + uses: actions/cache@v3 + with: + path: ${{ matrix.projectPath }}/Library + key: Library-2-${{ matrix.projectPath }}-${{ matrix.unityVersion }} + restore-keys: | + Library-2-${{ matrix.projectPath }}-${{ matrix.unityVersion }} + + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + # See the registration rationale above testGroupDefault's matching step. ~Claude + - name: Register Unity license problem matcher + run: echo "::add-matcher::.github/matchers/unity-license.json" + - name: Unity tests (license-contention retry) + uses: ./.github/actions/unity-test-with-retry + with: + unityVersion: ${{ matrix.unityVersion }} + testMode: ${{ matrix.testMode }} + projectPath: ${{ matrix.projectPath }} + customImage: ${{ env.DOCKER_REGISTRY }}/beamable/editor:ubuntu-${{ matrix.unityVersion }}-base-${{ env.GAME_CI_VERSION }} + customParameters: -warnaserror+ -executeMethod Beamable.Editor.BeamCli.BeamCliUtil.InstallToolFromLocalPackageSource + artifactsPath: ${{ matrix.testMode }}-${{ matrix.unityVersion }}-artifacts checkName: ${{ matrix.unityVersion }} ${{ matrix.testMode }} Test Results + unityEmail: ${{ secrets.UNITY_EMAIL_BUILD }} + unityPassword: ${{ secrets.UNITY_PASSWORD_BUILD }} + unitySerial: ${{ secrets.UNITY_SERIAL_BUILD }} + githubToken: ${{ secrets.GITHUB_TOKEN }} + + # See the annotation rationale above testGroupDefault's matching step. ~Claude + - name: Annotate Unity license contention + if: failure() + shell: bash + env: + ARTIFACTS: ${{ matrix.testMode }}-${{ matrix.unityVersion }}-artifacts + run: | + if find "$ARTIFACTS" -name '*.xml' 2>/dev/null | grep -q .; then + echo "Test result XML present: this is a genuine test failure, not license contention." + else + echo "::error title=Unity license contention::${{ matrix.unityVersion }} ${{ matrix.testMode }} produced no test results -- the editor exited before tests ran, the signature of a Unity license/seat activation failure (serial contention), not a test failure." + fi build: name: Build for ${{ matrix.targetPlatform }} @@ -340,6 +501,10 @@ jobs: cancel-in-progress: true strategy: fail-fast: false + # All device builds share the UNITY_SERIAL_DEVOPS seat (see the comment above + # testGroupDefault). max-parallel: 1 leaves a seat of headroom for the license + # return/activate transition. ~Claude + max-parallel: 1 matrix: targetPlatform: - Android @@ -350,15 +515,6 @@ jobs: - 6000.0.37f1 projectPath: - client - include: - - targetPlatform: Android - unityEmailOverride: UNITY_EMAIL_DEVOPS - unityPasswordOverride: UNITY_PASSWORD_DEVOPS - unitySerialOverride: UNITY_SERIAL_DEVOPS - - targetPlatform: StandaloneOSX - unityEmailOverride: UNITY_EMAIL_BUILD - unityPasswordOverride: UNITY_PASSWORD_BUILD - unitySerialOverride: UNITY_SERIAL_BUILD steps: - uses: actions/checkout@v4 with: @@ -446,9 +602,9 @@ jobs: - uses: game-ci/unity-builder@v4 timeout-minutes: 60 env: - UNITY_EMAIL: ${{ secrets[matrix.unityEmailOverride] || secrets.UNITY_EMAIL }} - UNITY_PASSWORD: ${{ secrets[matrix.unityPasswordOverride] || secrets.UNITY_PASSWORD }} - UNITY_SERIAL: ${{ secrets[matrix.unitySerialOverride] || secrets.UNITY_SERIAL }} + UNITY_EMAIL: ${{ secrets.UNITY_EMAIL_DEVOPS }} + UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD_DEVOPS }} + UNITY_SERIAL: ${{ secrets.UNITY_SERIAL_DEVOPS }} with: customImage: ${{ env.DOCKER_REGISTRY }}/beamable/editor:ubuntu-${{ matrix.unityVersion }}-${{steps.unityMap.outputs.UNITY_PLATFORM}}-3.1.0 targetPlatform: ${{ matrix.targetPlatform }}