ci(macrobenchmark): Run startup benchmark on Sauce Labs (POC)

The sentry-uitest-android-macrobenchmark module currently only runs on a
locally connected device. This wires it to Sauce Labs so we can evaluate
whether the cold-start timeToInitialDisplay benchmark can run on the
real-device cloud already used by our other benchmarks.

This is a proof-of-concept, gated behind workflow_dispatch and kept off the
per-PR path. Device-guard errors are intentionally not suppressed: on
non-rooted, unlocked-clock cloud devices the guards (UNLOCKED, DEBUGGABLE,
...) are expected to fire, and seeing which ones fire is the point of the
POC. It also tells us whether timeToInitialDisplay can be retrieved from
Sauce artifacts (benchmark JSON) or must be parsed from the device log.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: runningcode

.github/workflows/integration-tests-macrobenchmark.yml

@@ -0,0 +1,60 @@
+name: 'Integration Tests - Macrobenchmark (POC)'
+# Proof-of-concept: run the sentry-uitest-android-macrobenchmark cold-start benchmark on Sauce
+# Labs real devices. Manual trigger only — this is not yet a per-PR gate. The goal is to learn
+# (a) whether Macrobenchmark runs on non-rooted cloud devices and which device guards fire, and
+# (b) whether timeToInitialDisplay can be retrieved from Sauce artifacts. See the module README.
+on:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  macrobenchmark:
+    name: Macrobenchmark
+    runs-on: ubuntu-latest
+
+    # we copy the secret to the env variable in order to access it in the workflow
+    env:
+      SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
+      GRADLE_ENCRYPTION_KEY: ${{ secrets.GRADLE_ENCRYPTION_KEY }}
+
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          submodules: 'recursive'
+
+      - name: 'Set up Java: 17'
+        uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0
+        with:
+          cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }}
+
+      - name: Make assembleMacrobenchmark
+        if: env.SAUCE_USERNAME != null
+        run: make assembleMacrobenchmark
+
+      - name: Run Macrobenchmark in SauceLab
+        uses: saucelabs/saucectl-run-action@bc81720eb01738d9c664b07fe42621bd0014283f # pin@v3
+        if: env.SAUCE_USERNAME != null
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          sauce-username: ${{ secrets.SAUCE_USERNAME }}
+          sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }}
+          config-file: .sauce/sentry-uitest-android-macrobenchmark.yml
+
+      - name: Upload Sauce artifacts
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: macrobenchmark-artifacts
+          path: ./artifacts/
+          if-no-files-found: warn

.sauce/sentry-uitest-android-macrobenchmark.yml

@@ -0,0 +1,43 @@
+apiVersion: v1alpha
+kind: espresso
+sauce:
+  region: us-west-1
+  concurrency: 1
+  metadata:
+    build: sentry-uitest-android-macrobenchmark-$GITHUB_REF-$GITHUB_SHA
+    tags:
+      - benchmarks
+      - android
+      - macrobenchmark
+
+defaults:
+  timeout: 40m
+
+espresso:
+  # Target app under test: Macrobenchmark cold-starts sentry-samples-android. It must be
+  # release-like; the release build type is signed with the debug key so it installs on Sauce.
+  app: ./sentry-samples/sentry-samples-android/build/outputs/apk/release/sentry-samples-android-release.apk
+  # Instrumentation APK: the self-instrumenting com.android.test macrobenchmark module.
+  testApp: ./sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/build/outputs/apk/benchmark/sentry-uitest-android-macrobenchmark-benchmark.apk
+
+suites:
+
+  - name: "Macrobenchmark startup (api 35)"
+    # No test orchestrator and no clearPackageData: Macrobenchmark manages its own process
+    # restarts and AOT compilation, and StartupMode.COLD intentionally keeps app data and
+    # permissions (it force-stops rather than `pm clear`).
+    devices:
+      - id: Google_Pixel_9_Pro_XL_15_real_sjc1 # Google Pixel 9 Pro XL - api 35 (15) - high end
+
+# Grab both the benchmark JSON (if Sauce collects additional test output) and the device log, so
+# we can see which path actually yields timeToInitialDisplay. Macrobenchmark also logs guard
+# failures (UNLOCKED, DEBUGGABLE, ...) here, which is the expected first-run signal on cloud
+# hardware — see the module README.
+artifacts:
+  download:
+    when: always
+    match:
+      - junit.xml
+      - "*.log"
+      - "*-benchmarkData.json"
+    directory: ./artifacts/

Makefile

@@ -1,7 +1,8 @@
-.PHONY: all clean compile javadocs dryRelease update checkFormat api assembleBenchmarkTestRelease assembleUiTestRelease assembleUiTestCriticalRelease runUiTestCritical setupPython systemTest systemTestInteractive check preMerge publish
+.PHONY: all clean compile javadocs dryRelease update checkFormat api assembleBenchmarkTestRelease assembleMacrobenchmarkRelease assembleUiTestRelease assembleUiTestCriticalRelease runUiTestCritical setupPython systemTest systemTestInteractive check preMerge publish
 
 all: stop clean javadocs compile
 assembleBenchmarks: assembleBenchmarkTestRelease
+assembleMacrobenchmark: assembleMacrobenchmarkRelease
 assembleUiTests: assembleUiTestRelease
 preMerge: check
 publish: clean dryRelease
@@ -39,6 +40,10 @@ api:
 assembleBenchmarkTestRelease:
 	./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleAndroidTest
 
+# Assemble the target sample app (release) and the Macrobenchmark instrumentation apk
+assembleMacrobenchmarkRelease:
+	./gradlew :sentry-samples:sentry-samples-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-macrobenchmark:assembleBenchmark
+
 # Assemble release and Android test apk of the uitest-android module
 assembleUiTestRelease:
 	./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android:assembleAndroidTest