From 5acdd0a25c42eca5310bc2a1dac1c777f70ecb97 Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Thu, 2 Jul 2026 19:25:42 +0200 Subject: [PATCH 01/15] ci(macrobenchmark): Run startup benchmark on Sauce Labs (POC) The sentry-uitest-android-macrobenchmark module currently only runs on a locally connected device. This wires it to Sauce Labs so we can evaluate whether the cold-start timeToInitialDisplay benchmark can run on the real-device cloud already used by our other benchmarks. This is a proof-of-concept, gated behind workflow_dispatch and kept off the per-PR path. Device-guard errors are intentionally not suppressed: on non-rooted, unlocked-clock cloud devices the guards (UNLOCKED, DEBUGGABLE, ...) are expected to fire, and seeing which ones fire is the point of the POC. It also tells us whether timeToInitialDisplay can be retrieved from Sauce artifacts (benchmark JSON) or must be parsed from the device log. Co-Authored-By: Claude Opus 4.8 --- .../integration-tests-macrobenchmark.yml | 60 +++++++++++++++++++ .../sentry-uitest-android-macrobenchmark.yml | 43 +++++++++++++ Makefile | 7 ++- 3 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/integration-tests-macrobenchmark.yml create mode 100644 .sauce/sentry-uitest-android-macrobenchmark.yml diff --git a/.github/workflows/integration-tests-macrobenchmark.yml b/.github/workflows/integration-tests-macrobenchmark.yml new file mode 100644 index 00000000000..5fc25110a83 --- /dev/null +++ b/.github/workflows/integration-tests-macrobenchmark.yml @@ -0,0 +1,60 @@ +name: 'Integration Tests - Macrobenchmark (POC)' +# Proof-of-concept: run the sentry-uitest-android-macrobenchmark cold-start benchmark on Sauce +# Labs real devices. Manual trigger only — this is not yet a per-PR gate. The goal is to learn +# (a) whether Macrobenchmark runs on non-rooted cloud devices and which device guards fire, and +# (b) whether timeToInitialDisplay can be retrieved from Sauce artifacts. See the module README. +on: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + macrobenchmark: + name: Macrobenchmark + runs-on: ubuntu-latest + + # we copy the secret to the env variable in order to access it in the workflow + env: + SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} + GRADLE_ENCRYPTION_KEY: ${{ secrets.GRADLE_ENCRYPTION_KEY }} + + steps: + - name: Git checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + submodules: 'recursive' + + - name: 'Set up Java: 17' + uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5 + with: + distribution: 'temurin' + java-version: '17' + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 + with: + cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} + + - name: Make assembleMacrobenchmark + if: env.SAUCE_USERNAME != null + run: make assembleMacrobenchmark + + - name: Run Macrobenchmark in SauceLab + uses: saucelabs/saucectl-run-action@bc81720eb01738d9c664b07fe42621bd0014283f # pin@v3 + if: env.SAUCE_USERNAME != null + env: + GITHUB_TOKEN: ${{ github.token }} + with: + sauce-username: ${{ secrets.SAUCE_USERNAME }} + sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} + config-file: .sauce/sentry-uitest-android-macrobenchmark.yml + + - name: Upload Sauce artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: macrobenchmark-artifacts + path: ./artifacts/ + if-no-files-found: warn diff --git a/.sauce/sentry-uitest-android-macrobenchmark.yml b/.sauce/sentry-uitest-android-macrobenchmark.yml new file mode 100644 index 00000000000..6751560dc35 --- /dev/null +++ b/.sauce/sentry-uitest-android-macrobenchmark.yml @@ -0,0 +1,43 @@ +apiVersion: v1alpha +kind: espresso +sauce: + region: us-west-1 + concurrency: 1 + metadata: + build: sentry-uitest-android-macrobenchmark-$GITHUB_REF-$GITHUB_SHA + tags: + - benchmarks + - android + - macrobenchmark + +defaults: + timeout: 40m + +espresso: + # Target app under test: Macrobenchmark cold-starts sentry-samples-android. It must be + # release-like; the release build type is signed with the debug key so it installs on Sauce. + app: ./sentry-samples/sentry-samples-android/build/outputs/apk/release/sentry-samples-android-release.apk + # Instrumentation APK: the self-instrumenting com.android.test macrobenchmark module. + testApp: ./sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/build/outputs/apk/benchmark/sentry-uitest-android-macrobenchmark-benchmark.apk + +suites: + + - name: "Macrobenchmark startup (api 35)" + # No test orchestrator and no clearPackageData: Macrobenchmark manages its own process + # restarts and AOT compilation, and StartupMode.COLD intentionally keeps app data and + # permissions (it force-stops rather than `pm clear`). + devices: + - id: Google_Pixel_9_Pro_XL_15_real_sjc1 # Google Pixel 9 Pro XL - api 35 (15) - high end + +# Grab both the benchmark JSON (if Sauce collects additional test output) and the device log, so +# we can see which path actually yields timeToInitialDisplay. Macrobenchmark also logs guard +# failures (UNLOCKED, DEBUGGABLE, ...) here, which is the expected first-run signal on cloud +# hardware — see the module README. +artifacts: + download: + when: always + match: + - junit.xml + - "*.log" + - "*-benchmarkData.json" + directory: ./artifacts/ diff --git a/Makefile b/Makefile index 3967ff856ad..09413b6af30 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ -.PHONY: all clean compile javadocs dryRelease update checkFormat api assembleBenchmarkTestRelease assembleUiTestRelease assembleUiTestCriticalRelease runUiTestCritical setupPython systemTest systemTestInteractive check preMerge publish +.PHONY: all clean compile javadocs dryRelease update checkFormat api assembleBenchmarkTestRelease assembleMacrobenchmarkRelease assembleUiTestRelease assembleUiTestCriticalRelease runUiTestCritical setupPython systemTest systemTestInteractive check preMerge publish all: stop clean javadocs compile assembleBenchmarks: assembleBenchmarkTestRelease +assembleMacrobenchmark: assembleMacrobenchmarkRelease assembleUiTests: assembleUiTestRelease preMerge: check publish: clean dryRelease @@ -39,6 +40,10 @@ api: assembleBenchmarkTestRelease: ./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleAndroidTest +# Assemble the target sample app (release) and the Macrobenchmark instrumentation apk +assembleMacrobenchmarkRelease: + ./gradlew :sentry-samples:sentry-samples-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-macrobenchmark:assembleBenchmark + # Assemble release and Android test apk of the uitest-android module assembleUiTestRelease: ./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android:assembleAndroidTest From 233b4303de1b5fca16385f60a56c8f317c94adf4 Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Thu, 2 Jul 2026 19:31:18 +0200 Subject: [PATCH 02/15] ci(macrobenchmark): Call gradle directly instead of a Makefile target Drop the assembleMacrobenchmark Makefile target and invoke the two gradle assemble tasks directly from the workflow. Only one workflow uses them, so the extra indirection isn't worth it. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/integration-tests-macrobenchmark.yml | 4 ++-- Makefile | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration-tests-macrobenchmark.yml b/.github/workflows/integration-tests-macrobenchmark.yml index 5fc25110a83..bcab966e3b3 100644 --- a/.github/workflows/integration-tests-macrobenchmark.yml +++ b/.github/workflows/integration-tests-macrobenchmark.yml @@ -37,9 +37,9 @@ jobs: with: cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - - name: Make assembleMacrobenchmark + - name: Assemble target app and Macrobenchmark apk if: env.SAUCE_USERNAME != null - run: make assembleMacrobenchmark + run: ./gradlew :sentry-samples:sentry-samples-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-macrobenchmark:assembleBenchmark - name: Run Macrobenchmark in SauceLab uses: saucelabs/saucectl-run-action@bc81720eb01738d9c664b07fe42621bd0014283f # pin@v3 diff --git a/Makefile b/Makefile index 09413b6af30..3967ff856ad 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,7 @@ -.PHONY: all clean compile javadocs dryRelease update checkFormat api assembleBenchmarkTestRelease assembleMacrobenchmarkRelease assembleUiTestRelease assembleUiTestCriticalRelease runUiTestCritical setupPython systemTest systemTestInteractive check preMerge publish +.PHONY: all clean compile javadocs dryRelease update checkFormat api assembleBenchmarkTestRelease assembleUiTestRelease assembleUiTestCriticalRelease runUiTestCritical setupPython systemTest systemTestInteractive check preMerge publish all: stop clean javadocs compile assembleBenchmarks: assembleBenchmarkTestRelease -assembleMacrobenchmark: assembleMacrobenchmarkRelease assembleUiTests: assembleUiTestRelease preMerge: check publish: clean dryRelease @@ -40,10 +39,6 @@ api: assembleBenchmarkTestRelease: ./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleAndroidTest -# Assemble the target sample app (release) and the Macrobenchmark instrumentation apk -assembleMacrobenchmarkRelease: - ./gradlew :sentry-samples:sentry-samples-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-macrobenchmark:assembleBenchmark - # Assemble release and Android test apk of the uitest-android module assembleUiTestRelease: ./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android:assembleAndroidTest From 7e237f52103552f5216e180275852ede5812b4e1 Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Thu, 6 Aug 2026 17:13:49 +0200 Subject: [PATCH 03/15] ci(macrobenchmark): Pull benchmark results via Real Device Access API saucectl cannot retrieve Macrobenchmark's output. Its artifacts.download.match filters a hardcoded list of assets Sauce hosts for a job -- device.log, junit.xml, video.mp4, network.har, crash.json, screenshots.zip -- and never reads the device filesystem, so benchmarkData.json and the per-iteration perfetto traces stay stranded on the device. The POC's "*-benchmarkData.json" match line could never have matched anything; drop it. Drive the run through the Real Device Access API instead, which exposes adb shell and pullFile: reserve a device, install both APKs, run the instrumentation, then pull the results off. This also recovers the perfetto traces, the only way to resolve sub-millisecond SDK changes that timeToInitialDisplay cannot see. The API is an open-beta add-on, so the workflow probes entitlement before spending a Gradle build on it, and defaults to a one-iteration smoke run. --- .../integration-tests-macrobenchmark.yml | 81 +++- .../sentry-uitest-android-macrobenchmark.yml | 9 +- scripts/macrobenchmark-sauce.py | 355 ++++++++++++++++++ 3 files changed, 422 insertions(+), 23 deletions(-) create mode 100755 scripts/macrobenchmark-sauce.py diff --git a/.github/workflows/integration-tests-macrobenchmark.yml b/.github/workflows/integration-tests-macrobenchmark.yml index bcab966e3b3..34f5c3888e6 100644 --- a/.github/workflows/integration-tests-macrobenchmark.yml +++ b/.github/workflows/integration-tests-macrobenchmark.yml @@ -1,10 +1,31 @@ -name: 'Integration Tests - Macrobenchmark (POC)' -# Proof-of-concept: run the sentry-uitest-android-macrobenchmark cold-start benchmark on Sauce -# Labs real devices. Manual trigger only — this is not yet a per-PR gate. The goal is to learn -# (a) whether Macrobenchmark runs on non-rooted cloud devices and which device guards fire, and -# (b) whether timeToInitialDisplay can be retrieved from Sauce artifacts. See the module README. +name: 'Integration Tests - Macrobenchmark' +# Runs the sentry-uitest-android-macrobenchmark cold-start benchmark on a Sauce Labs real +# device via the Real Device Access API, and pulls benchmarkData.json plus the per-iteration +# perfetto traces off the device. saucectl cannot do the pulling: it only downloads assets +# Sauce hosts for a job, never files from the device. See scripts/macrobenchmark-sauce.py. +# +# Manual trigger only -- this reports numbers, it does not gate PRs. Cloud devices have +# unlocked CPU clocks, so run-to-run spread is far wider than most SDK-init changes. on: workflow_dispatch: + inputs: + mode: + description: 'smoke = 1 iteration without AOT compilation (fast, validates the plumbing); full = the benchmark as declared' + type: choice + options: + - smoke + - full + default: smoke + device-name: + description: 'Sauce device id or regex' + required: false + default: 'Google_Pixel_9_Pro_XL_15_real.*' + # Temporary scaffolding: workflow_dispatch cannot target a workflow that does not exist on + # the default branch yet, so trigger on pushes to this branch while we validate the flow. + # Remove before merge. + push: + branches: + - no/macrobenchmark-sauce-results concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -18,43 +39,67 @@ jobs: # we copy the secret to the env variable in order to access it in the workflow env: SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} + SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} GRADLE_ENCRYPTION_KEY: ${{ secrets.GRADLE_ENCRYPTION_KEY }} + # Defaults apply to the push trigger too, where `inputs` is empty. + MODE: ${{ inputs.mode || 'smoke' }} + DEVICE_NAME: ${{ inputs.device-name || 'Google_Pixel_9_Pro_XL_15_real.*' }} steps: - name: Git checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: submodules: 'recursive' + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.10.5' + + - name: Install Python dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -r requirements.txt + - name: 'Set up Java: 17' - uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5 + uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5 with: distribution: 'temurin' java-version: '17' - name: Setup Gradle - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 + uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.3.0 with: cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} + # Probe first: the Real Device Access API is an open-beta add-on, so a missing + # entitlement should fail in seconds rather than after a full Gradle build. + - name: Check Real Device Access API entitlement + if: env.SAUCE_USERNAME != null + run: python3 scripts/macrobenchmark-sauce.py --probe-only --device-name "$DEVICE_NAME" + - name: Assemble target app and Macrobenchmark apk if: env.SAUCE_USERNAME != null run: ./gradlew :sentry-samples:sentry-samples-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-macrobenchmark:assembleBenchmark - - name: Run Macrobenchmark in SauceLab - uses: saucelabs/saucectl-run-action@bc81720eb01738d9c664b07fe42621bd0014283f # pin@v3 + - name: Run Macrobenchmark on Sauce Labs if: env.SAUCE_USERNAME != null - env: - GITHUB_TOKEN: ${{ github.token }} - with: - sauce-username: ${{ secrets.SAUCE_USERNAME }} - sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} - config-file: .sauce/sentry-uitest-android-macrobenchmark.yml + run: | + if [ "$MODE" = "smoke" ]; then + mode_args=(--iterations 1 --skip-compilation) + else + mode_args=() + fi + python3 scripts/macrobenchmark-sauce.py \ + --app sentry-samples/sentry-samples-android/build/outputs/apk/release/sentry-samples-android-release.apk \ + --test-app sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/build/outputs/apk/benchmark/sentry-uitest-android-macrobenchmark-benchmark.apk \ + --device-name "$DEVICE_NAME" \ + --out-dir ./artifacts/macrobenchmark \ + "${mode_args[@]}" - - name: Upload Sauce artifacts + - name: Upload benchmark results uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: always() with: - name: macrobenchmark-artifacts + name: macrobenchmark-results path: ./artifacts/ if-no-files-found: warn diff --git a/.sauce/sentry-uitest-android-macrobenchmark.yml b/.sauce/sentry-uitest-android-macrobenchmark.yml index 6751560dc35..4b45388c8d9 100644 --- a/.sauce/sentry-uitest-android-macrobenchmark.yml +++ b/.sauce/sentry-uitest-android-macrobenchmark.yml @@ -29,15 +29,14 @@ suites: devices: - id: Google_Pixel_9_Pro_XL_15_real_sjc1 # Google Pixel 9 Pro XL - api 35 (15) - high end -# Grab both the benchmark JSON (if Sauce collects additional test output) and the device log, so -# we can see which path actually yields timeToInitialDisplay. Macrobenchmark also logs guard -# failures (UNLOCKED, DEBUGGABLE, ...) here, which is the expected first-run signal on cloud -# hardware — see the module README. +# The device log carries the actual results: Sauce only returns assets it produces itself, so it +# cannot pull Macrobenchmark's `-benchmarkData.json` off the device. SentryStartupBenchmark +# echoes that JSON into logcat instead, and scripts/parse-macrobenchmark-log.py reassembles it. +# The log also holds Macrobenchmark's device guard warnings (unlocked clocks, low battery, ...). artifacts: download: when: always match: - junit.xml - "*.log" - - "*-benchmarkData.json" directory: ./artifacts/ diff --git a/scripts/macrobenchmark-sauce.py b/scripts/macrobenchmark-sauce.py new file mode 100755 index 00000000000..e4322791f89 --- /dev/null +++ b/scripts/macrobenchmark-sauce.py @@ -0,0 +1,355 @@ +#!/usr/bin/env python3 + +""" +Run the Android Macrobenchmark on a Sauce Labs real device and pull its results back. + +Why this exists instead of a saucectl espresso suite: saucectl can only download the +assets Sauce itself hosts for a job -- device.log, junit.xml, video.mp4, network.har, +crash.json, screenshots.zip -- and never reads the device filesystem. Macrobenchmark +writes its numbers to `-benchmarkData.json` and its per-iteration perfetto traces +into on-device storage, so under saucectl they stay stranded on the device. + +The Real Device Access API (open beta) does expose the device: it offers adb shell and +a pullFile endpoint. So this script drives the run itself -- reserve a device, install +both APKs, run the instrumentation, pull the results -- which also gets us the perfetto +traces, the only way to resolve sub-millisecond SDK changes that timeToInitialDisplay +cannot see. + +Usage: + # Just check whether this account has Real Device Access API entitlement + python3 scripts/macrobenchmark-sauce.py --probe-only + + # Fast smoke test: 1 iteration, no AOT compilation + python3 scripts/macrobenchmark-sauce.py \ + --app sentry-samples/sentry-samples-android/build/outputs/apk/release/sentry-samples-android-release.apk \ + --test-app sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/build/outputs/apk/benchmark/sentry-uitest-android-macrobenchmark-benchmark.apk \ + --iterations 1 --skip-compilation + + # Full run, as the benchmark declares it + python3 scripts/macrobenchmark-sauce.py --app --test-app + +Requires SAUCE_USERNAME and SAUCE_ACCESS_KEY in the environment. +""" + +import argparse +import json +import os +import sys +import time +from pathlib import Path + +import requests + +TEST_PACKAGE = "io.sentry.uitest.android.macrobenchmark" +TEST_RUNNER = "androidx.test.runner.AndroidJUnitRunner" + +# Macrobenchmark's default output dir. Deliberately not overridden via +# additionalTestOutputDir: on API 29+ the test process cannot write outside its own +# scoped directories, and this media dir is the one place both the app and the shell +# can reach -- which is exactly why androidx.benchmark picked it (see b/181601156), +# and it is what makes it reachable via pullFile. +DEVICE_OUTPUT_DIR = f"/sdcard/Android/media/{TEST_PACKAGE}" + +# Shell-owned scratch space for the instrumentation's own stdout and exit code. +SHELL_SCRATCH_DIR = "/data/local/tmp/macrobenchmark" + +DEFAULT_DEVICE = "Google_Pixel_9_Pro_XL_15_real.*" + + +class SauceError(Exception): + pass + + +class RealDeviceSession: + """Thin client for the Real Device Access API, scoped to one device session.""" + + def __init__(self, region, username, access_key): + self.api = f"https://api.{region}.saucelabs.com" + self.rda = f"{self.api}/rdc/v2" + self.auth = (username, access_key) + self.session_id = None + + def _request(self, method, url, expect=(200,), **kwargs): + response = requests.request(method, url, auth=self.auth, timeout=120, **kwargs) + if response.status_code not in expect: + raise SauceError( + f"{method} {url} returned {response.status_code}: {response.text[:500]}" + ) + return response + + # --- entitlement probe ------------------------------------------------- + + def probe(self, device_name): + """Confirms the account can reach the API, and that the device exists.""" + response = requests.get( + f"{self.rda}/devices", + params={"deviceId": device_name}, + auth=self.auth, + timeout=60, + ) + if response.status_code in (401, 403): + raise SauceError( + "Real Device Access API rejected these credentials " + f"({response.status_code}). It is an open-beta add-on, so the account " + "most likely lacks entitlement -- ask Sauce to enable it." + ) + if response.status_code != 200: + raise SauceError( + f"GET /devices returned {response.status_code}: {response.text[:500]}" + ) + return [d for d in response.json() if d.get("os") == "ANDROID"] + + # --- app storage ------------------------------------------------------- + + def upload_app(self, apk): + """Uploads an APK to App Storage and returns its `storage:` reference.""" + with open(apk, "rb") as payload: + response = self._request( + "POST", + f"{self.api}/v1/storage/upload", + files={"payload": (apk.name, payload)}, + data={"name": apk.name}, + ) + return "storage:" + response.json()["item"]["id"] + + # --- session lifecycle ------------------------------------------------- + + def open(self, device_name, duration="PT1H"): + body = { + "device": {"deviceName": device_name, "os": "android"}, + "configuration": {"sessionDuration": duration}, + } + self.session_id = self._request("POST", f"{self.rda}/sessions", json=body).json()["id"] + print(f"Session {self.session_id} requested on {device_name}") + self._await_state("ACTIVE") + return self.session_id + + def _await_state(self, wanted, timeout=600): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + state = self._request("GET", f"{self.rda}/sessions/{self.session_id}").json()["state"] + if state == wanted: + return + if state in ("CLOSED", "ERRORED"): + raise SauceError(f"Session entered {state} while waiting for {wanted}") + time.sleep(5) + raise SauceError(f"Session did not reach {wanted} within {timeout}s") + + def close(self): + if not self.session_id: + return + # Best effort: a leaked session holds the device until sessionDuration expires. + try: + self._request( + "DELETE", f"{self.rda}/sessions/{self.session_id}", expect=(200, 204, 404) + ) + print(f"Session {self.session_id} closed") + except (SauceError, requests.RequestException) as error: + print(f"WARNING: failed to close session {self.session_id}: {error}") + + # --- device interactions ---------------------------------------------- + + def _device(self, endpoint, **kwargs): + return self._request( + "POST", f"{self.rda}/sessions/{self.session_id}/device/{endpoint}", **kwargs + ) + + def shell(self, command): + return self._device("executeShellCommand", json={"adbShellCommand": command}).json()[ + "stdout" + ] + + def install(self, app_reference, timeout=600): + # enableInstrumentation=false keeps the APKs byte-identical: Sauce's + # instrumentation re-signs and hooks the app, which would mean benchmarking + # something other than what we built. We need none of the features it unlocks. + self._device( + "installApp", json={"app": app_reference, "enableInstrumentation": False} + ) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + installations = self._device("listAppInstallations").json()["appInstallations"] + status = next( + (i for i in installations if i.get("app") == app_reference), None + ) + if status and status["status"] == "FINISHED": + print(f"Installed {app_reference}") + return + if status and status["status"] == "ERROR": + raise SauceError(f"Installation of {app_reference} failed") + time.sleep(5) + raise SauceError(f"Installation of {app_reference} did not finish within {timeout}s") + + def disable_animations(self): + self._device("applySettings", json={"animations": False}, expect=(204,)) + + def list_files(self, path): + return self._device("listFiles", json={"path": path}).json() + + def pull_file(self, path): + return self._device("pullFile", json={"path": path}).content + + +def instrumentation_args(iterations, skip_compilation): + """Builds the `-e key value` arguments for `am instrument`. + + Note we never pass androidx.benchmark.dryRunMode.enable: dry-run mode forces + outputEnable to false (Arguments.kt), so it writes no benchmarkData.json at all + and cannot verify that retrieval works. For a fast check use one iteration with + compilation disabled instead -- that still produces real output. + """ + args = [] + if iterations is not None: + args += ["androidx.benchmark.iterations", str(iterations)] + if skip_compilation: + args += ["androidx.benchmark.compilation.enabled", "false"] + return " ".join(f"-e {key} {value}" for key, value in zip(args[::2], args[1::2])) + + +def run_benchmark(device, iterations, skip_compilation, timeout): + """Starts the instrumentation detached and waits for it to finish.""" + # Leading rm is belt and braces -- Outputs also clears its dir on startup. Separated by + # `;` so a missing dir on a fresh device does not stop the mkdir. + device.shell(f"rm -rf {SHELL_SCRATCH_DIR} {DEVICE_OUTPUT_DIR}; mkdir -p {SHELL_SCRATCH_DIR}") + + stdout_file = f"{SHELL_SCRATCH_DIR}/instrumentation.txt" + exit_code_file = f"{SHELL_SCRATCH_DIR}/exitcode" + instrumentation = ( + f"am instrument -w -r {instrumentation_args(iterations, skip_compilation)} " + f"{TEST_PACKAGE}/{TEST_RUNNER}" + ) + # Detached, because executeShellCommand is documented to time out on long-running + # commands (504 "Please do not execute long running adb commands") and a cold-start + # benchmark runs for minutes. The exit code file is the completion signal. + device.shell( + f"nohup sh -c '{instrumentation} > {stdout_file} 2>&1; " + f"echo $? > {exit_code_file}' > /dev/null 2>&1 &" + ) + print(f"Started: {instrumentation}") + + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if device.shell(f"cat {exit_code_file} 2>/dev/null").strip(): + break + time.sleep(15) + else: + raise SauceError(f"Benchmark did not finish within {timeout}s") + + exit_code = device.shell(f"cat {exit_code_file}").strip() + output = device.shell(f"cat {stdout_file}") + print(f"--- instrumentation output (exit {exit_code}) ---\n{output}") + return exit_code, output + + +def pull_results(device, out_dir): + """Pulls the benchmark JSON and perfetto traces out of the device output dir.""" + out_dir.mkdir(parents=True, exist_ok=True) + try: + entries = device.list_files(DEVICE_OUTPUT_DIR) + except SauceError as error: + raise SauceError( + f"Could not list {DEVICE_OUTPUT_DIR}: {error}. The benchmark likely never " + "wrote results -- check the instrumentation output above." + ) + + pulled = [] + for entry in entries: + name = entry.rsplit("/", 1)[-1] + if not (name.endswith(".json") or name.endswith(".perfetto-trace")): + continue + path = entry if entry.startswith("/") else f"{DEVICE_OUTPUT_DIR}/{name}" + target = out_dir / name + target.write_bytes(device.pull_file(path)) + pulled.append(target) + print(f"Pulled {path} -> {target}") + + if not pulled: + raise SauceError(f"No results found in {DEVICE_OUTPUT_DIR} (saw: {entries})") + return pulled + + +def print_summary(benchmark_data): + data = json.loads(benchmark_data.read_text()) + context = data["context"] + build = context["build"] + print( + f"\n{build['brand']} {build['model']} (api {build['version']['sdk']}), " + f"compilation {context['compilationMode']}, cpuLocked={context['cpuLocked']}" + ) + for benchmark in data["benchmarks"]: + for metric, result in sorted(benchmark["metrics"].items()): + print( + f" {benchmark['name']} {metric}: " + f"min {result['minimum']:.1f} / median {result['median']:.1f} / " + f"max {result['maximum']:.1f} " + f"(CoV {result['coefficientOfVariation'] * 100:.1f}%, " + f"{len(result['runs'])} iterations)" + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--app", type=Path, help="target app APK (sentry-samples-android release)") + parser.add_argument("--test-app", type=Path, help="macrobenchmark instrumentation APK") + parser.add_argument("--device-name", default=DEFAULT_DEVICE, help="device id or regex") + parser.add_argument("--region", default="us-west-1") + parser.add_argument("--iterations", type=int, help="override the benchmark's iteration count") + parser.add_argument( + "--skip-compilation", + action="store_true", + help="skip AOT compilation; much faster, but the numbers are not comparable", + ) + parser.add_argument("--out-dir", type=Path, default=Path("artifacts/macrobenchmark")) + parser.add_argument("--timeout", type=int, default=2400, help="seconds to wait for the run") + parser.add_argument( + "--probe-only", + action="store_true", + help="only check API entitlement and device availability, then exit", + ) + args = parser.parse_args() + + username = os.environ.get("SAUCE_USERNAME") + access_key = os.environ.get("SAUCE_ACCESS_KEY") + if not username or not access_key: + sys.exit("SAUCE_USERNAME and SAUCE_ACCESS_KEY must be set") + + device = RealDeviceSession(args.region, username, access_key) + + try: + matches = device.probe(args.device_name) + except SauceError as error: + sys.exit(f"Real Device Access API probe failed: {error}") + + print(f"Real Device Access API reachable; {len(matches)} Android device(s) match " + f"{args.device_name!r}: {[d['id'] for d in matches[:5]]}") + if args.probe_only: + return + if not matches: + sys.exit(f"No Android device matches {args.device_name!r}") + if not args.app or not args.test_app: + sys.exit("--app and --test-app are required unless --probe-only is given") + + app_reference = device.upload_app(args.app) + test_app_reference = device.upload_app(args.test_app) + + try: + device.open(args.device_name) + device.install(app_reference) + device.install(test_app_reference) + device.disable_animations() + + exit_code, _ = run_benchmark(device, args.iterations, args.skip_compilation, args.timeout) + results = pull_results(device, args.out_dir) + finally: + device.close() + + for result in results: + if result.name.endswith("-benchmarkData.json"): + print_summary(result) + + if exit_code != "0": + sys.exit(f"Instrumentation exited with {exit_code}") + + +if __name__ == "__main__": + main() From 9128c6f72465484033988cf99142e46c536b57d0 Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Thu, 6 Aug 2026 17:18:13 +0200 Subject: [PATCH 04/15] ci(macrobenchmark): Fix app upload status and drop the iterations override App Storage upload answers 201, not 200, so the strict status check rejected every upload before a device was even reserved. Also drop --iterations: androidx.benchmark.iterations only feeds the microbenchmark path (BenchmarkStateLegacy, MicrobenchmarkPhase). Macrobenchmark reads its iteration count from the test source and ignores the argument, so skipping AOT compilation is the only way to shorten a run that still writes results -- dryRunMode would shorten it but forces outputEnable to false, producing no benchmarkData.json to retrieve. Track installations by installationId rather than by the app reference, which Sauce may echo back normalised. --- .../integration-tests-macrobenchmark.yml | 4 +- scripts/macrobenchmark-sauce.py | 49 +++++++++++-------- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/.github/workflows/integration-tests-macrobenchmark.yml b/.github/workflows/integration-tests-macrobenchmark.yml index 34f5c3888e6..bcf5780a828 100644 --- a/.github/workflows/integration-tests-macrobenchmark.yml +++ b/.github/workflows/integration-tests-macrobenchmark.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: inputs: mode: - description: 'smoke = 1 iteration without AOT compilation (fast, validates the plumbing); full = the benchmark as declared' + description: 'smoke = skip AOT compilation (faster, validates the plumbing); full = the benchmark as declared' type: choice options: - smoke @@ -85,7 +85,7 @@ jobs: if: env.SAUCE_USERNAME != null run: | if [ "$MODE" = "smoke" ]; then - mode_args=(--iterations 1 --skip-compilation) + mode_args=(--skip-compilation) else mode_args=() fi diff --git a/scripts/macrobenchmark-sauce.py b/scripts/macrobenchmark-sauce.py index e4322791f89..e06af6bb7f1 100755 --- a/scripts/macrobenchmark-sauce.py +++ b/scripts/macrobenchmark-sauce.py @@ -19,11 +19,11 @@ # Just check whether this account has Real Device Access API entitlement python3 scripts/macrobenchmark-sauce.py --probe-only - # Fast smoke test: 1 iteration, no AOT compilation + # Faster smoke test: same iterations, but no AOT compilation python3 scripts/macrobenchmark-sauce.py \ --app sentry-samples/sentry-samples-android/build/outputs/apk/release/sentry-samples-android-release.apk \ --test-app sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/build/outputs/apk/benchmark/sentry-uitest-android-macrobenchmark-benchmark.apk \ - --iterations 1 --skip-compilation + --skip-compilation # Full run, as the benchmark declares it python3 scripts/macrobenchmark-sauce.py --app --test-app @@ -107,10 +107,13 @@ def upload_app(self, apk): response = self._request( "POST", f"{self.api}/v1/storage/upload", + expect=(200, 201), files={"payload": (apk.name, payload)}, data={"name": apk.name}, ) - return "storage:" + response.json()["item"]["id"] + reference = "storage:" + response.json()["item"]["id"] + print(f"Uploaded {apk.name} as {reference}") + return reference # --- session lifecycle ------------------------------------------------- @@ -163,14 +166,17 @@ def install(self, app_reference, timeout=600): # enableInstrumentation=false keeps the APKs byte-identical: Sauce's # instrumentation re-signs and hooks the app, which would mean benchmarking # something other than what we built. We need none of the features it unlocks. - self._device( + started = self._device( "installApp", json={"app": app_reference, "enableInstrumentation": False} - ) + ).json() + # Track by installationId rather than the app reference, which Sauce may echo back + # in a normalised form. + installation_id = started["installationId"] deadline = time.monotonic() + timeout while time.monotonic() < deadline: installations = self._device("listAppInstallations").json()["appInstallations"] status = next( - (i for i in installations if i.get("app") == app_reference), None + (i for i in installations if i.get("installationId") == installation_id), None ) if status and status["status"] == "FINISHED": print(f"Installed {app_reference}") @@ -190,23 +196,25 @@ def pull_file(self, path): return self._device("pullFile", json={"path": path}).content -def instrumentation_args(iterations, skip_compilation): +def instrumentation_args(skip_compilation): """Builds the `-e key value` arguments for `am instrument`. - Note we never pass androidx.benchmark.dryRunMode.enable: dry-run mode forces - outputEnable to false (Arguments.kt), so it writes no benchmarkData.json at all - and cannot verify that retrieval works. For a fast check use one iteration with - compilation disabled instead -- that still produces real output. + Deliberately not offered here: + + - androidx.benchmark.dryRunMode.enable would cut the run to one iteration, but it + also forces outputEnable to false (Arguments.kt), so no benchmarkData.json is + written at all -- a dry run cannot verify that retrieval works. + - androidx.benchmark.iterations only feeds the *micro*benchmark path + (BenchmarkStateLegacy, MicrobenchmarkPhase). Macrobenchmark takes its iteration + count from the test source and ignores the argument, so skipping AOT compilation + is the only way to shorten a run while still producing results. """ - args = [] - if iterations is not None: - args += ["androidx.benchmark.iterations", str(iterations)] - if skip_compilation: - args += ["androidx.benchmark.compilation.enabled", "false"] - return " ".join(f"-e {key} {value}" for key, value in zip(args[::2], args[1::2])) + if not skip_compilation: + return "" + return "-e androidx.benchmark.compilation.enabled false" -def run_benchmark(device, iterations, skip_compilation, timeout): +def run_benchmark(device, skip_compilation, timeout): """Starts the instrumentation detached and waits for it to finish.""" # Leading rm is belt and braces -- Outputs also clears its dir on startup. Separated by # `;` so a missing dir on a fresh device does not stop the mkdir. @@ -215,7 +223,7 @@ def run_benchmark(device, iterations, skip_compilation, timeout): stdout_file = f"{SHELL_SCRATCH_DIR}/instrumentation.txt" exit_code_file = f"{SHELL_SCRATCH_DIR}/exitcode" instrumentation = ( - f"am instrument -w -r {instrumentation_args(iterations, skip_compilation)} " + f"am instrument -w -r {instrumentation_args(skip_compilation)} " f"{TEST_PACKAGE}/{TEST_RUNNER}" ) # Detached, because executeShellCommand is documented to time out on long-running @@ -293,7 +301,6 @@ def main(): parser.add_argument("--test-app", type=Path, help="macrobenchmark instrumentation APK") parser.add_argument("--device-name", default=DEFAULT_DEVICE, help="device id or regex") parser.add_argument("--region", default="us-west-1") - parser.add_argument("--iterations", type=int, help="override the benchmark's iteration count") parser.add_argument( "--skip-compilation", action="store_true", @@ -338,7 +345,7 @@ def main(): device.install(test_app_reference) device.disable_animations() - exit_code, _ = run_benchmark(device, args.iterations, args.skip_compilation, args.timeout) + exit_code, _ = run_benchmark(device, args.skip_compilation, args.timeout) results = pull_results(device, args.out_dir) finally: device.close() From 559c872afb0f111d38e607c16bcc290ac87cc9a0 Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Thu, 6 Aug 2026 17:30:22 +0200 Subject: [PATCH 05/15] ci(macrobenchmark): Resolve the device from the catalog before opening a session The first run reached the API but matched zero devices: Real Device Access descriptors are not the ids used in .sauce/*.yml, which carry an OS version and a region suffix (Google_Pixel_9_Pro_XL_15_real_sjc1). Fetch the catalog unfiltered and match client-side against both descriptor id and name, then open the session with the resolved concrete id rather than a pattern. On no match, print every available Android device so a naming mismatch is diagnosable from a single run -- the endpoint's own deviceId filter cannot distinguish "no devices on this account" from "your pattern matched nothing". Also run the no-match check before --probe-only returns. It previously came after, so the probe reported success and the real run failed on it minutes later. --- .../integration-tests-macrobenchmark.yml | 4 +- scripts/macrobenchmark-sauce.py | 75 +++++++++++++++---- 2 files changed, 61 insertions(+), 18 deletions(-) diff --git a/.github/workflows/integration-tests-macrobenchmark.yml b/.github/workflows/integration-tests-macrobenchmark.yml index bcf5780a828..b1012935c63 100644 --- a/.github/workflows/integration-tests-macrobenchmark.yml +++ b/.github/workflows/integration-tests-macrobenchmark.yml @@ -19,7 +19,7 @@ on: device-name: description: 'Sauce device id or regex' required: false - default: 'Google_Pixel_9_Pro_XL_15_real.*' + default: 'Google_Pixel_9_Pro_XL' # Temporary scaffolding: workflow_dispatch cannot target a workflow that does not exist on # the default branch yet, so trigger on pushes to this branch while we validate the flow. # Remove before merge. @@ -43,7 +43,7 @@ jobs: GRADLE_ENCRYPTION_KEY: ${{ secrets.GRADLE_ENCRYPTION_KEY }} # Defaults apply to the push trigger too, where `inputs` is empty. MODE: ${{ inputs.mode || 'smoke' }} - DEVICE_NAME: ${{ inputs.device-name || 'Google_Pixel_9_Pro_XL_15_real.*' }} + DEVICE_NAME: ${{ inputs.device-name || 'Google_Pixel_9_Pro_XL' }} steps: - name: Git checkout diff --git a/scripts/macrobenchmark-sauce.py b/scripts/macrobenchmark-sauce.py index e06af6bb7f1..3b289ef612c 100755 --- a/scripts/macrobenchmark-sauce.py +++ b/scripts/macrobenchmark-sauce.py @@ -34,6 +34,7 @@ import argparse import json import os +import re import sys import time from pathlib import Path @@ -53,7 +54,10 @@ # Shell-owned scratch space for the instrumentation's own stdout and exit code. SHELL_SCRATCH_DIR = "/data/local/tmp/macrobenchmark" -DEFAULT_DEVICE = "Google_Pixel_9_Pro_XL_15_real.*" +# Matched as a regex against descriptor ids and names. Kept loose on purpose: Real Device +# Access descriptors are not the ids used in .sauce/*.yml, which carry an OS version and a +# region suffix (Google_Pixel_9_Pro_XL_15_real_sjc1). +DEFAULT_DEVICE = "Google_Pixel_9_Pro_XL" class SauceError(Exception): @@ -79,14 +83,14 @@ def _request(self, method, url, expect=(200,), **kwargs): # --- entitlement probe ------------------------------------------------- - def probe(self, device_name): - """Confirms the account can reach the API, and that the device exists.""" - response = requests.get( - f"{self.rda}/devices", - params={"deviceId": device_name}, - auth=self.auth, - timeout=60, - ) + def device_catalog(self): + """Returns the full device catalog, confirming the account can reach the API. + + Deliberately unfiltered: the endpoint's own deviceId filter gives no way to tell + "this account has no devices" apart from "your pattern matched nothing", and the + descriptors here are not the ids saucectl uses (no region suffix). + """ + response = requests.get(f"{self.rda}/devices", auth=self.auth, timeout=60) if response.status_code in (401, 403): raise SauceError( "Real Device Access API rejected these credentials " @@ -97,7 +101,7 @@ def probe(self, device_name): raise SauceError( f"GET /devices returned {response.status_code}: {response.text[:500]}" ) - return [d for d in response.json() if d.get("os") == "ANDROID"] + return response.json() # --- app storage ------------------------------------------------------- @@ -118,6 +122,8 @@ def upload_app(self, apk): # --- session lifecycle ------------------------------------------------- def open(self, device_name, duration="PT1H"): + # device_name is a concrete descriptor id resolved from the catalog, not a pattern, + # so we do not depend on the server's regex semantics. body = { "device": {"deviceName": device_name, "os": "android"}, "configuration": {"sessionDuration": duration}, @@ -276,6 +282,39 @@ def pull_results(device, out_dir): return pulled +def android_devices(catalog): + return [d for d in catalog if str(d.get("os", "")).upper() == "ANDROID"] + + +def matching_devices(catalog, pattern): + """Android descriptors whose id or human-readable name matches `pattern`.""" + regex = re.compile(pattern, re.IGNORECASE) + return [ + d + for d in android_devices(catalog) + if regex.search(d.get("id", "")) or regex.search(d.get("name", "")) + ] + + +def describe_catalog(catalog, pattern): + android = android_devices(catalog) + lines = [ + f"Real Device Access API reachable: {len(catalog)} device(s) in the catalog, " + f"{len(android)} Android." + ] + matches = matching_devices(catalog, pattern) + lines.append(f"{len(matches)} match {pattern!r}: {[d['id'] for d in matches[:10]]}") + if not matches: + # Print the catalog so a naming mismatch is diagnosable from one run: RDA + # descriptors have no region suffix, unlike the ids in .sauce/*.yml. + lines.append("Available Android devices:") + lines += [ + f" {d.get('id')} ({d.get('name')}, {d.get('os')} {d.get('osVersion')})" + for d in sorted(android, key=lambda d: d.get("id", "")) + ] + return "\n".join(lines) + + def print_summary(benchmark_data): data = json.loads(benchmark_data.read_text()) context = data["context"] @@ -323,16 +362,20 @@ def main(): device = RealDeviceSession(args.region, username, access_key) try: - matches = device.probe(args.device_name) + catalog = device.device_catalog() except SauceError as error: sys.exit(f"Real Device Access API probe failed: {error}") - print(f"Real Device Access API reachable; {len(matches)} Android device(s) match " - f"{args.device_name!r}: {[d['id'] for d in matches[:5]]}") - if args.probe_only: - return + print(describe_catalog(catalog, args.device_name)) + matches = matching_devices(catalog, args.device_name) + # Checked before --probe-only returns, so the probe fails loudly on a device name that + # matches nothing rather than passing and letting the real run discover it. if not matches: sys.exit(f"No Android device matches {args.device_name!r}") + device_id = matches[0]["id"] + print(f"Using device {device_id}") + if args.probe_only: + return if not args.app or not args.test_app: sys.exit("--app and --test-app are required unless --probe-only is given") @@ -340,7 +383,7 @@ def main(): test_app_reference = device.upload_app(args.test_app) try: - device.open(args.device_name) + device.open(device_id) device.install(app_reference) device.install(test_app_reference) device.disable_animations() From 1f99849d43cfe2e69dba17dc7f4c708c9b2dcaa1 Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Thu, 6 Aug 2026 17:35:10 +0200 Subject: [PATCH 06/15] ci(macrobenchmark): Probe every region when the device catalog is empty An empty catalog answers 200 exactly like a populated one, so the first probe reported the API as reachable while there was in fact nothing to run on. Report device availability and the other regions' catalogs too, so an unentitled account is distinguishable from a misconfigured region. --- scripts/macrobenchmark-sauce.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/macrobenchmark-sauce.py b/scripts/macrobenchmark-sauce.py index 3b289ef612c..3a045e9d5b6 100755 --- a/scripts/macrobenchmark-sauce.py +++ b/scripts/macrobenchmark-sauce.py @@ -59,6 +59,8 @@ # region suffix (Google_Pixel_9_Pro_XL_15_real_sjc1). DEFAULT_DEVICE = "Google_Pixel_9_Pro_XL" +ALL_REGIONS = ("us-west-1", "eu-central-1", "us-east-4") + class SauceError(Exception): pass @@ -103,6 +105,13 @@ def device_catalog(self): ) return response.json() + def device_statuses(self): + """Device availability, which is a separate endpoint from the catalog.""" + response = requests.get(f"{self.rda}/devices/status", auth=self.auth, timeout=60) + if response.status_code != 200: + return f"HTTP {response.status_code}: {response.text[:200]}" + return response.json() + # --- app storage ------------------------------------------------------- def upload_app(self, apk): @@ -367,6 +376,21 @@ def main(): sys.exit(f"Real Device Access API probe failed: {error}") print(describe_catalog(catalog, args.device_name)) + + # An empty catalog answers 200 just like a populated one, so distinguish "no devices + # entitled to this API" from "wrong region" before concluding the API is unusable. + if not catalog: + statuses = device.device_statuses() + count = len(statuses) if isinstance(statuses, list) else statuses + print(f"{args.region}: /devices/status -> {count}") + for region in ALL_REGIONS: + if region == args.region: + continue + other = RealDeviceSession(region, username, access_key) + try: + print(f"{region}: {len(other.device_catalog())} device(s) in the catalog") + except SauceError as error: + print(f"{region}: {error}") matches = matching_devices(catalog, args.device_name) # Checked before --probe-only returns, so the probe fails loudly on a device name that # matches nothing rather than passing and letting the real run discover it. From b8e8b84cdeb209893c1072a696ae35caed929b28 Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Thu, 6 Aug 2026 17:44:12 +0200 Subject: [PATCH 07/15] ci(macrobenchmark): Resolve devices from /devices/status, not /devices On a public-cloud account /devices answers 200 with an empty list, which made the probe look like a missing entitlement. /devices/status returns the whole fleet -- 392 devices, all public, 368 available -- and its descriptors are exactly the ids already used in .sauce/*.yml, so the original Google_Pixel_9_Pro_XL_15_real_sjc1 is back as the default. Prefer AVAILABLE devices when a pattern matches several, and report each match's state so an IN_USE device explains itself. --- .../integration-tests-macrobenchmark.yml | 4 +- scripts/macrobenchmark-sauce.py | 99 +++++++------------ 2 files changed, 36 insertions(+), 67 deletions(-) diff --git a/.github/workflows/integration-tests-macrobenchmark.yml b/.github/workflows/integration-tests-macrobenchmark.yml index b1012935c63..b7dc54661ea 100644 --- a/.github/workflows/integration-tests-macrobenchmark.yml +++ b/.github/workflows/integration-tests-macrobenchmark.yml @@ -19,7 +19,7 @@ on: device-name: description: 'Sauce device id or regex' required: false - default: 'Google_Pixel_9_Pro_XL' + default: 'Google_Pixel_9_Pro_XL_15_real_sjc1' # Temporary scaffolding: workflow_dispatch cannot target a workflow that does not exist on # the default branch yet, so trigger on pushes to this branch while we validate the flow. # Remove before merge. @@ -43,7 +43,7 @@ jobs: GRADLE_ENCRYPTION_KEY: ${{ secrets.GRADLE_ENCRYPTION_KEY }} # Defaults apply to the push trigger too, where `inputs` is empty. MODE: ${{ inputs.mode || 'smoke' }} - DEVICE_NAME: ${{ inputs.device-name || 'Google_Pixel_9_Pro_XL' }} + DEVICE_NAME: ${{ inputs.device-name || 'Google_Pixel_9_Pro_XL_15_real_sjc1' }} steps: - name: Git checkout diff --git a/scripts/macrobenchmark-sauce.py b/scripts/macrobenchmark-sauce.py index 3a045e9d5b6..f5ae7e2c461 100755 --- a/scripts/macrobenchmark-sauce.py +++ b/scripts/macrobenchmark-sauce.py @@ -54,12 +54,9 @@ # Shell-owned scratch space for the instrumentation's own stdout and exit code. SHELL_SCRATCH_DIR = "/data/local/tmp/macrobenchmark" -# Matched as a regex against descriptor ids and names. Kept loose on purpose: Real Device -# Access descriptors are not the ids used in .sauce/*.yml, which carry an OS version and a -# region suffix (Google_Pixel_9_Pro_XL_15_real_sjc1). -DEFAULT_DEVICE = "Google_Pixel_9_Pro_XL" - -ALL_REGIONS = ("us-west-1", "eu-central-1", "us-east-4") +# Matched as a regex against /devices/status descriptors, which are the same ids used in +# .sauce/*.yml. Same high-end device the existing benchmark suite uses. +DEFAULT_DEVICE = "Google_Pixel_9_Pro_XL_15_real_sjc1" class SauceError(Exception): @@ -85,14 +82,14 @@ def _request(self, method, url, expect=(200,), **kwargs): # --- entitlement probe ------------------------------------------------- - def device_catalog(self): - """Returns the full device catalog, confirming the account can reach the API. + def devices(self): + """Returns the device fleet with per-device availability. - Deliberately unfiltered: the endpoint's own deviceId filter gives no way to tell - "this account has no devices" apart from "your pattern matched nothing", and the - descriptors here are not the ids saucectl uses (no region suffix). + Uses /devices/status rather than /devices: on a public-cloud account the catalog + endpoint answers 200 with an empty list, while this one returns the whole fleet, + and its descriptors are the same ids used in .sauce/*.yml. """ - response = requests.get(f"{self.rda}/devices", auth=self.auth, timeout=60) + response = requests.get(f"{self.rda}/devices/status", auth=self.auth, timeout=60) if response.status_code in (401, 403): raise SauceError( "Real Device Access API rejected these credentials " @@ -101,16 +98,9 @@ def device_catalog(self): ) if response.status_code != 200: raise SauceError( - f"GET /devices returned {response.status_code}: {response.text[:500]}" + f"GET /devices/status returned {response.status_code}: {response.text[:500]}" ) - return response.json() - - def device_statuses(self): - """Device availability, which is a separate endpoint from the catalog.""" - response = requests.get(f"{self.rda}/devices/status", auth=self.auth, timeout=60) - if response.status_code != 200: - return f"HTTP {response.status_code}: {response.text[:200]}" - return response.json() + return response.json()["devices"] # --- app storage ------------------------------------------------------- @@ -291,36 +281,30 @@ def pull_results(device, out_dir): return pulled -def android_devices(catalog): - return [d for d in catalog if str(d.get("os", "")).upper() == "ANDROID"] +def matching_devices(devices, pattern): + """Devices whose descriptor matches `pattern`, free ones first. - -def matching_devices(catalog, pattern): - """Android descriptors whose id or human-readable name matches `pattern`.""" + /devices/status carries no OS field, so the pattern is the only OS filter -- which is + fine, since the caller names a concrete device. + """ regex = re.compile(pattern, re.IGNORECASE) - return [ - d - for d in android_devices(catalog) - if regex.search(d.get("id", "")) or regex.search(d.get("name", "")) - ] + matches = [d for d in devices if regex.search(d.get("descriptor", ""))] + return sorted(matches, key=lambda d: d.get("state") != "AVAILABLE") -def describe_catalog(catalog, pattern): - android = android_devices(catalog) +def describe_devices(devices, pattern): + available = [d for d in devices if d.get("state") == "AVAILABLE"] + matches = matching_devices(devices, pattern) lines = [ - f"Real Device Access API reachable: {len(catalog)} device(s) in the catalog, " - f"{len(android)} Android." + f"Real Device Access API reachable: {len(devices)} device(s), " + f"{len(available)} available.", + f"{len(matches)} match {pattern!r}: " + + ", ".join(f"{d['descriptor']} ({d.get('state')})" for d in matches[:10]), ] - matches = matching_devices(catalog, pattern) - lines.append(f"{len(matches)} match {pattern!r}: {[d['id'] for d in matches[:10]]}") if not matches: - # Print the catalog so a naming mismatch is diagnosable from one run: RDA - # descriptors have no region suffix, unlike the ids in .sauce/*.yml. - lines.append("Available Android devices:") - lines += [ - f" {d.get('id')} ({d.get('name')}, {d.get('os')} {d.get('osVersion')})" - for d in sorted(android, key=lambda d: d.get("id", "")) - ] + # Print the fleet so a naming mismatch is diagnosable from one run. + lines.append("Available devices:") + lines += [f" {d['descriptor']}" for d in sorted(available, key=lambda d: d["descriptor"])] return "\n".join(lines) @@ -371,33 +355,18 @@ def main(): device = RealDeviceSession(args.region, username, access_key) try: - catalog = device.device_catalog() + fleet = device.devices() except SauceError as error: sys.exit(f"Real Device Access API probe failed: {error}") - print(describe_catalog(catalog, args.device_name)) - - # An empty catalog answers 200 just like a populated one, so distinguish "no devices - # entitled to this API" from "wrong region" before concluding the API is unusable. - if not catalog: - statuses = device.device_statuses() - count = len(statuses) if isinstance(statuses, list) else statuses - print(f"{args.region}: /devices/status -> {count}") - for region in ALL_REGIONS: - if region == args.region: - continue - other = RealDeviceSession(region, username, access_key) - try: - print(f"{region}: {len(other.device_catalog())} device(s) in the catalog") - except SauceError as error: - print(f"{region}: {error}") - matches = matching_devices(catalog, args.device_name) + print(describe_devices(fleet, args.device_name)) + matches = matching_devices(fleet, args.device_name) # Checked before --probe-only returns, so the probe fails loudly on a device name that # matches nothing rather than passing and letting the real run discover it. if not matches: - sys.exit(f"No Android device matches {args.device_name!r}") - device_id = matches[0]["id"] - print(f"Using device {device_id}") + sys.exit(f"No device matches {args.device_name!r}") + device_id = matches[0]["descriptor"] + print(f"Using device {device_id} ({matches[0].get('state')})") if args.probe_only: return if not args.app or not args.test_app: From 4d56521a03c00a2ec355f1fba3f1e3099693a61f Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Thu, 6 Aug 2026 17:55:55 +0200 Subject: [PATCH 08/15] ci(macrobenchmark): Recover results through logcat, not the device API The Real Device Access API can pull files off a device, but it cannot reserve one on a public-cloud account: POST /sessions rejects every request with deviceClasses=[PRIVATE_DEVICE], and the request body has no way to ask for a public device. Everything up to reservation worked -- App Storage upload, device resolution from /devices/status, the session request itself -- so this is a commercial limit, not a technical one. Revert to saucectl and drop the script; git history has it if we ever lease private devices, since that route would also return the per-iteration perfetto traces. Instead have the benchmark echo its own benchmarkData.json into logcat in chunks, which reaches CI as device.log, the one asset Sauce does hand back. scripts/parse-macrobenchmark-log.py reassembles it, validates the JSON, and writes a timeToInitialDisplay table to the job summary. --- .../integration-tests-macrobenchmark.yml | 71 ++-- scripts/macrobenchmark-sauce.py | 398 ------------------ scripts/parse-macrobenchmark-log.py | 117 +++++ .../macrobenchmark/SentryStartupBenchmark.kt | 62 ++- 4 files changed, 200 insertions(+), 448 deletions(-) delete mode 100755 scripts/macrobenchmark-sauce.py create mode 100755 scripts/parse-macrobenchmark-log.py diff --git a/.github/workflows/integration-tests-macrobenchmark.yml b/.github/workflows/integration-tests-macrobenchmark.yml index b7dc54661ea..f75afdbc73e 100644 --- a/.github/workflows/integration-tests-macrobenchmark.yml +++ b/.github/workflows/integration-tests-macrobenchmark.yml @@ -1,25 +1,18 @@ name: 'Integration Tests - Macrobenchmark' # Runs the sentry-uitest-android-macrobenchmark cold-start benchmark on a Sauce Labs real -# device via the Real Device Access API, and pulls benchmarkData.json plus the per-iteration -# perfetto traces off the device. saucectl cannot do the pulling: it only downloads assets -# Sauce hosts for a job, never files from the device. See scripts/macrobenchmark-sauce.py. +# device and recovers timeToInitialDisplay from the device log. +# +# Getting the numbers back is the awkward part. saucectl only downloads assets Sauce hosts +# for a job (device.log, junit.xml, video.mp4, ...) and never reads the device filesystem, +# so Macrobenchmark's benchmarkData.json cannot be fetched directly. The Real Device Access +# API can pull device files, but POST /sessions accepts only private devices +# (deviceClasses=[PRIVATE_DEVICE]) and we run on the public cloud. So the benchmark echoes +# its own results into logcat, and scripts/parse-macrobenchmark-log.py reassembles them. # # Manual trigger only -- this reports numbers, it does not gate PRs. Cloud devices have # unlocked CPU clocks, so run-to-run spread is far wider than most SDK-init changes. on: workflow_dispatch: - inputs: - mode: - description: 'smoke = skip AOT compilation (faster, validates the plumbing); full = the benchmark as declared' - type: choice - options: - - smoke - - full - default: smoke - device-name: - description: 'Sauce device id or regex' - required: false - default: 'Google_Pixel_9_Pro_XL_15_real_sjc1' # Temporary scaffolding: workflow_dispatch cannot target a workflow that does not exist on # the default branch yet, so trigger on pushes to this branch while we validate the flow. # Remove before merge. @@ -39,11 +32,7 @@ jobs: # we copy the secret to the env variable in order to access it in the workflow env: SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} - SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} GRADLE_ENCRYPTION_KEY: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - # Defaults apply to the push trigger too, where `inputs` is empty. - MODE: ${{ inputs.mode || 'smoke' }} - DEVICE_NAME: ${{ inputs.device-name || 'Google_Pixel_9_Pro_XL_15_real_sjc1' }} steps: - name: Git checkout @@ -51,15 +40,6 @@ jobs: with: submodules: 'recursive' - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.10.5' - - - name: Install Python dependencies - run: | - python3 -m pip install --upgrade pip - python3 -m pip install -r requirements.txt - - name: 'Set up Java: 17' uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5 with: @@ -71,32 +51,29 @@ jobs: with: cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - # Probe first: the Real Device Access API is an open-beta add-on, so a missing - # entitlement should fail in seconds rather than after a full Gradle build. - - name: Check Real Device Access API entitlement - if: env.SAUCE_USERNAME != null - run: python3 scripts/macrobenchmark-sauce.py --probe-only --device-name "$DEVICE_NAME" - - name: Assemble target app and Macrobenchmark apk if: env.SAUCE_USERNAME != null run: ./gradlew :sentry-samples:sentry-samples-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-macrobenchmark:assembleBenchmark - - name: Run Macrobenchmark on Sauce Labs + - name: Run Macrobenchmark in SauceLab + uses: saucelabs/saucectl-run-action@283660aa934c02723c497efa151d582a3acc5801 # pin@v3 if: env.SAUCE_USERNAME != null + env: + GITHUB_TOKEN: ${{ github.token }} + with: + sauce-username: ${{ secrets.SAUCE_USERNAME }} + sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} + config-file: .sauce/sentry-uitest-android-macrobenchmark.yml + + # Runs even when the suite fails: a failed benchmark still logs whatever it managed to + # measure, and the parser's own error explains what was missing. + - name: Recover benchmark results from the device log + if: always() && env.SAUCE_USERNAME != null run: | - if [ "$MODE" = "smoke" ]; then - mode_args=(--skip-compilation) - else - mode_args=() - fi - python3 scripts/macrobenchmark-sauce.py \ - --app sentry-samples/sentry-samples-android/build/outputs/apk/release/sentry-samples-android-release.apk \ - --test-app sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/build/outputs/apk/benchmark/sentry-uitest-android-macrobenchmark-benchmark.apk \ - --device-name "$DEVICE_NAME" \ - --out-dir ./artifacts/macrobenchmark \ - "${mode_args[@]}" + python3 scripts/parse-macrobenchmark-log.py ./artifacts \ + --json-out ./artifacts/benchmarkData.json | tee -a "$GITHUB_STEP_SUMMARY" - - name: Upload benchmark results + - name: Upload Sauce artifacts uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: always() with: diff --git a/scripts/macrobenchmark-sauce.py b/scripts/macrobenchmark-sauce.py deleted file mode 100755 index f5ae7e2c461..00000000000 --- a/scripts/macrobenchmark-sauce.py +++ /dev/null @@ -1,398 +0,0 @@ -#!/usr/bin/env python3 - -""" -Run the Android Macrobenchmark on a Sauce Labs real device and pull its results back. - -Why this exists instead of a saucectl espresso suite: saucectl can only download the -assets Sauce itself hosts for a job -- device.log, junit.xml, video.mp4, network.har, -crash.json, screenshots.zip -- and never reads the device filesystem. Macrobenchmark -writes its numbers to `-benchmarkData.json` and its per-iteration perfetto traces -into on-device storage, so under saucectl they stay stranded on the device. - -The Real Device Access API (open beta) does expose the device: it offers adb shell and -a pullFile endpoint. So this script drives the run itself -- reserve a device, install -both APKs, run the instrumentation, pull the results -- which also gets us the perfetto -traces, the only way to resolve sub-millisecond SDK changes that timeToInitialDisplay -cannot see. - -Usage: - # Just check whether this account has Real Device Access API entitlement - python3 scripts/macrobenchmark-sauce.py --probe-only - - # Faster smoke test: same iterations, but no AOT compilation - python3 scripts/macrobenchmark-sauce.py \ - --app sentry-samples/sentry-samples-android/build/outputs/apk/release/sentry-samples-android-release.apk \ - --test-app sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/build/outputs/apk/benchmark/sentry-uitest-android-macrobenchmark-benchmark.apk \ - --skip-compilation - - # Full run, as the benchmark declares it - python3 scripts/macrobenchmark-sauce.py --app --test-app - -Requires SAUCE_USERNAME and SAUCE_ACCESS_KEY in the environment. -""" - -import argparse -import json -import os -import re -import sys -import time -from pathlib import Path - -import requests - -TEST_PACKAGE = "io.sentry.uitest.android.macrobenchmark" -TEST_RUNNER = "androidx.test.runner.AndroidJUnitRunner" - -# Macrobenchmark's default output dir. Deliberately not overridden via -# additionalTestOutputDir: on API 29+ the test process cannot write outside its own -# scoped directories, and this media dir is the one place both the app and the shell -# can reach -- which is exactly why androidx.benchmark picked it (see b/181601156), -# and it is what makes it reachable via pullFile. -DEVICE_OUTPUT_DIR = f"/sdcard/Android/media/{TEST_PACKAGE}" - -# Shell-owned scratch space for the instrumentation's own stdout and exit code. -SHELL_SCRATCH_DIR = "/data/local/tmp/macrobenchmark" - -# Matched as a regex against /devices/status descriptors, which are the same ids used in -# .sauce/*.yml. Same high-end device the existing benchmark suite uses. -DEFAULT_DEVICE = "Google_Pixel_9_Pro_XL_15_real_sjc1" - - -class SauceError(Exception): - pass - - -class RealDeviceSession: - """Thin client for the Real Device Access API, scoped to one device session.""" - - def __init__(self, region, username, access_key): - self.api = f"https://api.{region}.saucelabs.com" - self.rda = f"{self.api}/rdc/v2" - self.auth = (username, access_key) - self.session_id = None - - def _request(self, method, url, expect=(200,), **kwargs): - response = requests.request(method, url, auth=self.auth, timeout=120, **kwargs) - if response.status_code not in expect: - raise SauceError( - f"{method} {url} returned {response.status_code}: {response.text[:500]}" - ) - return response - - # --- entitlement probe ------------------------------------------------- - - def devices(self): - """Returns the device fleet with per-device availability. - - Uses /devices/status rather than /devices: on a public-cloud account the catalog - endpoint answers 200 with an empty list, while this one returns the whole fleet, - and its descriptors are the same ids used in .sauce/*.yml. - """ - response = requests.get(f"{self.rda}/devices/status", auth=self.auth, timeout=60) - if response.status_code in (401, 403): - raise SauceError( - "Real Device Access API rejected these credentials " - f"({response.status_code}). It is an open-beta add-on, so the account " - "most likely lacks entitlement -- ask Sauce to enable it." - ) - if response.status_code != 200: - raise SauceError( - f"GET /devices/status returned {response.status_code}: {response.text[:500]}" - ) - return response.json()["devices"] - - # --- app storage ------------------------------------------------------- - - def upload_app(self, apk): - """Uploads an APK to App Storage and returns its `storage:` reference.""" - with open(apk, "rb") as payload: - response = self._request( - "POST", - f"{self.api}/v1/storage/upload", - expect=(200, 201), - files={"payload": (apk.name, payload)}, - data={"name": apk.name}, - ) - reference = "storage:" + response.json()["item"]["id"] - print(f"Uploaded {apk.name} as {reference}") - return reference - - # --- session lifecycle ------------------------------------------------- - - def open(self, device_name, duration="PT1H"): - # device_name is a concrete descriptor id resolved from the catalog, not a pattern, - # so we do not depend on the server's regex semantics. - body = { - "device": {"deviceName": device_name, "os": "android"}, - "configuration": {"sessionDuration": duration}, - } - self.session_id = self._request("POST", f"{self.rda}/sessions", json=body).json()["id"] - print(f"Session {self.session_id} requested on {device_name}") - self._await_state("ACTIVE") - return self.session_id - - def _await_state(self, wanted, timeout=600): - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - state = self._request("GET", f"{self.rda}/sessions/{self.session_id}").json()["state"] - if state == wanted: - return - if state in ("CLOSED", "ERRORED"): - raise SauceError(f"Session entered {state} while waiting for {wanted}") - time.sleep(5) - raise SauceError(f"Session did not reach {wanted} within {timeout}s") - - def close(self): - if not self.session_id: - return - # Best effort: a leaked session holds the device until sessionDuration expires. - try: - self._request( - "DELETE", f"{self.rda}/sessions/{self.session_id}", expect=(200, 204, 404) - ) - print(f"Session {self.session_id} closed") - except (SauceError, requests.RequestException) as error: - print(f"WARNING: failed to close session {self.session_id}: {error}") - - # --- device interactions ---------------------------------------------- - - def _device(self, endpoint, **kwargs): - return self._request( - "POST", f"{self.rda}/sessions/{self.session_id}/device/{endpoint}", **kwargs - ) - - def shell(self, command): - return self._device("executeShellCommand", json={"adbShellCommand": command}).json()[ - "stdout" - ] - - def install(self, app_reference, timeout=600): - # enableInstrumentation=false keeps the APKs byte-identical: Sauce's - # instrumentation re-signs and hooks the app, which would mean benchmarking - # something other than what we built. We need none of the features it unlocks. - started = self._device( - "installApp", json={"app": app_reference, "enableInstrumentation": False} - ).json() - # Track by installationId rather than the app reference, which Sauce may echo back - # in a normalised form. - installation_id = started["installationId"] - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - installations = self._device("listAppInstallations").json()["appInstallations"] - status = next( - (i for i in installations if i.get("installationId") == installation_id), None - ) - if status and status["status"] == "FINISHED": - print(f"Installed {app_reference}") - return - if status and status["status"] == "ERROR": - raise SauceError(f"Installation of {app_reference} failed") - time.sleep(5) - raise SauceError(f"Installation of {app_reference} did not finish within {timeout}s") - - def disable_animations(self): - self._device("applySettings", json={"animations": False}, expect=(204,)) - - def list_files(self, path): - return self._device("listFiles", json={"path": path}).json() - - def pull_file(self, path): - return self._device("pullFile", json={"path": path}).content - - -def instrumentation_args(skip_compilation): - """Builds the `-e key value` arguments for `am instrument`. - - Deliberately not offered here: - - - androidx.benchmark.dryRunMode.enable would cut the run to one iteration, but it - also forces outputEnable to false (Arguments.kt), so no benchmarkData.json is - written at all -- a dry run cannot verify that retrieval works. - - androidx.benchmark.iterations only feeds the *micro*benchmark path - (BenchmarkStateLegacy, MicrobenchmarkPhase). Macrobenchmark takes its iteration - count from the test source and ignores the argument, so skipping AOT compilation - is the only way to shorten a run while still producing results. - """ - if not skip_compilation: - return "" - return "-e androidx.benchmark.compilation.enabled false" - - -def run_benchmark(device, skip_compilation, timeout): - """Starts the instrumentation detached and waits for it to finish.""" - # Leading rm is belt and braces -- Outputs also clears its dir on startup. Separated by - # `;` so a missing dir on a fresh device does not stop the mkdir. - device.shell(f"rm -rf {SHELL_SCRATCH_DIR} {DEVICE_OUTPUT_DIR}; mkdir -p {SHELL_SCRATCH_DIR}") - - stdout_file = f"{SHELL_SCRATCH_DIR}/instrumentation.txt" - exit_code_file = f"{SHELL_SCRATCH_DIR}/exitcode" - instrumentation = ( - f"am instrument -w -r {instrumentation_args(skip_compilation)} " - f"{TEST_PACKAGE}/{TEST_RUNNER}" - ) - # Detached, because executeShellCommand is documented to time out on long-running - # commands (504 "Please do not execute long running adb commands") and a cold-start - # benchmark runs for minutes. The exit code file is the completion signal. - device.shell( - f"nohup sh -c '{instrumentation} > {stdout_file} 2>&1; " - f"echo $? > {exit_code_file}' > /dev/null 2>&1 &" - ) - print(f"Started: {instrumentation}") - - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if device.shell(f"cat {exit_code_file} 2>/dev/null").strip(): - break - time.sleep(15) - else: - raise SauceError(f"Benchmark did not finish within {timeout}s") - - exit_code = device.shell(f"cat {exit_code_file}").strip() - output = device.shell(f"cat {stdout_file}") - print(f"--- instrumentation output (exit {exit_code}) ---\n{output}") - return exit_code, output - - -def pull_results(device, out_dir): - """Pulls the benchmark JSON and perfetto traces out of the device output dir.""" - out_dir.mkdir(parents=True, exist_ok=True) - try: - entries = device.list_files(DEVICE_OUTPUT_DIR) - except SauceError as error: - raise SauceError( - f"Could not list {DEVICE_OUTPUT_DIR}: {error}. The benchmark likely never " - "wrote results -- check the instrumentation output above." - ) - - pulled = [] - for entry in entries: - name = entry.rsplit("/", 1)[-1] - if not (name.endswith(".json") or name.endswith(".perfetto-trace")): - continue - path = entry if entry.startswith("/") else f"{DEVICE_OUTPUT_DIR}/{name}" - target = out_dir / name - target.write_bytes(device.pull_file(path)) - pulled.append(target) - print(f"Pulled {path} -> {target}") - - if not pulled: - raise SauceError(f"No results found in {DEVICE_OUTPUT_DIR} (saw: {entries})") - return pulled - - -def matching_devices(devices, pattern): - """Devices whose descriptor matches `pattern`, free ones first. - - /devices/status carries no OS field, so the pattern is the only OS filter -- which is - fine, since the caller names a concrete device. - """ - regex = re.compile(pattern, re.IGNORECASE) - matches = [d for d in devices if regex.search(d.get("descriptor", ""))] - return sorted(matches, key=lambda d: d.get("state") != "AVAILABLE") - - -def describe_devices(devices, pattern): - available = [d for d in devices if d.get("state") == "AVAILABLE"] - matches = matching_devices(devices, pattern) - lines = [ - f"Real Device Access API reachable: {len(devices)} device(s), " - f"{len(available)} available.", - f"{len(matches)} match {pattern!r}: " - + ", ".join(f"{d['descriptor']} ({d.get('state')})" for d in matches[:10]), - ] - if not matches: - # Print the fleet so a naming mismatch is diagnosable from one run. - lines.append("Available devices:") - lines += [f" {d['descriptor']}" for d in sorted(available, key=lambda d: d["descriptor"])] - return "\n".join(lines) - - -def print_summary(benchmark_data): - data = json.loads(benchmark_data.read_text()) - context = data["context"] - build = context["build"] - print( - f"\n{build['brand']} {build['model']} (api {build['version']['sdk']}), " - f"compilation {context['compilationMode']}, cpuLocked={context['cpuLocked']}" - ) - for benchmark in data["benchmarks"]: - for metric, result in sorted(benchmark["metrics"].items()): - print( - f" {benchmark['name']} {metric}: " - f"min {result['minimum']:.1f} / median {result['median']:.1f} / " - f"max {result['maximum']:.1f} " - f"(CoV {result['coefficientOfVariation'] * 100:.1f}%, " - f"{len(result['runs'])} iterations)" - ) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--app", type=Path, help="target app APK (sentry-samples-android release)") - parser.add_argument("--test-app", type=Path, help="macrobenchmark instrumentation APK") - parser.add_argument("--device-name", default=DEFAULT_DEVICE, help="device id or regex") - parser.add_argument("--region", default="us-west-1") - parser.add_argument( - "--skip-compilation", - action="store_true", - help="skip AOT compilation; much faster, but the numbers are not comparable", - ) - parser.add_argument("--out-dir", type=Path, default=Path("artifacts/macrobenchmark")) - parser.add_argument("--timeout", type=int, default=2400, help="seconds to wait for the run") - parser.add_argument( - "--probe-only", - action="store_true", - help="only check API entitlement and device availability, then exit", - ) - args = parser.parse_args() - - username = os.environ.get("SAUCE_USERNAME") - access_key = os.environ.get("SAUCE_ACCESS_KEY") - if not username or not access_key: - sys.exit("SAUCE_USERNAME and SAUCE_ACCESS_KEY must be set") - - device = RealDeviceSession(args.region, username, access_key) - - try: - fleet = device.devices() - except SauceError as error: - sys.exit(f"Real Device Access API probe failed: {error}") - - print(describe_devices(fleet, args.device_name)) - matches = matching_devices(fleet, args.device_name) - # Checked before --probe-only returns, so the probe fails loudly on a device name that - # matches nothing rather than passing and letting the real run discover it. - if not matches: - sys.exit(f"No device matches {args.device_name!r}") - device_id = matches[0]["descriptor"] - print(f"Using device {device_id} ({matches[0].get('state')})") - if args.probe_only: - return - if not args.app or not args.test_app: - sys.exit("--app and --test-app are required unless --probe-only is given") - - app_reference = device.upload_app(args.app) - test_app_reference = device.upload_app(args.test_app) - - try: - device.open(device_id) - device.install(app_reference) - device.install(test_app_reference) - device.disable_animations() - - exit_code, _ = run_benchmark(device, args.skip_compilation, args.timeout) - results = pull_results(device, args.out_dir) - finally: - device.close() - - for result in results: - if result.name.endswith("-benchmarkData.json"): - print_summary(result) - - if exit_code != "0": - sys.exit(f"Instrumentation exited with {exit_code}") - - -if __name__ == "__main__": - main() diff --git a/scripts/parse-macrobenchmark-log.py b/scripts/parse-macrobenchmark-log.py new file mode 100755 index 00000000000..3fad2122b9e --- /dev/null +++ b/scripts/parse-macrobenchmark-log.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Recover Macrobenchmark results from a Sauce Labs device log. + +Sauce Labs cannot pull arbitrary files off a real device, so +SentryStartupBenchmark echoes its `-benchmarkData.json` into logcat as +numbered chunks. This reassembles those chunks and prints a Markdown summary. + +Usage: + parse-macrobenchmark-log.py [--json-out benchmarkData.json] +""" + +import argparse +import json +import re +import sys +from pathlib import Path + +# Must match SentryStartupBenchmark.LOG_TAG and its "[index/total]" chunk prefix. +CHUNK_RE = re.compile(r"SentryBenchmarkData\s*:\s*\[(\d+)/(\d+)\](.*)$") + + +def collect_chunks(log_files): + """Returns the chunk texts keyed by index, plus the expected total.""" + chunks = {} + total = None + for log_file in log_files: + # Sauce device logs occasionally carry undecodable bytes; don't die on them. + for line in log_file.read_text(errors="replace").splitlines(): + match = CHUNK_RE.search(line) + if not match: + continue + index, chunk_total, text = int(match.group(1)), int(match.group(2)), match.group(3) + if total is not None and chunk_total != total: + sys.exit( + f"Found chunks from more than one benchmark run " + f"(totals {total} and {chunk_total}) in {log_file}" + ) + total = chunk_total + chunks[index] = text + return chunks, total + + +def reassemble(chunks, total): + missing = [i for i in range(1, total + 1) if i not in chunks] + if missing: + sys.exit(f"Incomplete benchmark data: missing chunk(s) {missing} of {total}") + return "".join(chunks[i] for i in range(1, total + 1)) + + +def format_summary(data): + context = data["context"] + build = context["build"] + lines = [ + "## Macrobenchmark results", + "", + f"**Device:** {build['brand']} {build['model']} " + f"(api {build['version']['sdk']}, {context['cpuCoreCount']} cores) · " + f"**compilation:** {context['compilationMode']} · " + f"**CPU clocks locked:** {context['cpuLocked']}", + "", + ] + + if not context["cpuLocked"]: + lines += [ + "> CPU clocks are unlocked on this device, so run-to-run spread is wide. " + "Treat these numbers as a trend, not a regression gate.", + "", + ] + + lines += ["| Benchmark | Metric | min | median | max | CoV | iterations |", "|---|---|--:|--:|--:|--:|--:|"] + for benchmark in data["benchmarks"]: + for metric, result in sorted(benchmark["metrics"].items()): + lines.append( + f"| `{benchmark['className'].rsplit('.', 1)[-1]}.{benchmark['name']}` " + f"| {metric} " + f"| {result['minimum']:.1f} " + f"| {result['median']:.1f} " + f"| {result['maximum']:.1f} " + f"| {result['coefficientOfVariation'] * 100:.1f}% " + f"| {len(result['runs'])} |" + ) + + for benchmark in data["benchmarks"]: + for metric, result in sorted(benchmark["metrics"].items()): + runs = ", ".join(f"{run:.1f}" for run in result["runs"]) + lines += ["", f"
{metric} per iteration", "", runs, "", "
"] + + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("artifacts_dir", type=Path, help="directory of downloaded Sauce artifacts") + parser.add_argument("--json-out", type=Path, help="where to write the recovered benchmarkData.json") + args = parser.parse_args() + + log_files = sorted(args.artifacts_dir.rglob("*.log")) + if not log_files: + sys.exit(f"No *.log files under {args.artifacts_dir}") + + chunks, total = collect_chunks(log_files) + if total is None: + sys.exit( + "No SentryBenchmarkData chunks in the device log. The benchmark most likely " + "failed before reporting — check junit.xml and the log for Macrobenchmark errors." + ) + + data = json.loads(reassemble(chunks, total)) + + if args.json_out: + args.json_out.write_text(json.dumps(data, indent=2)) + + print(format_summary(data)) + + +if __name__ == "__main__": + main() diff --git a/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt b/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt index ee49fe8beff..6099b45fa3d 100644 --- a/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt +++ b/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt @@ -1,5 +1,6 @@ package io.sentry.uitest.android.macrobenchmark +import android.util.Log import androidx.benchmark.macro.CompilationMode import androidx.benchmark.macro.ExperimentalMetricApi import androidx.benchmark.macro.StartupMode @@ -7,6 +8,9 @@ import androidx.benchmark.macro.StartupTimingMetric import androidx.benchmark.macro.TraceSectionMetric import androidx.benchmark.macro.junit4.MacrobenchmarkRule import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import java.io.File +import org.junit.AfterClass import org.junit.Rule import org.junit.Test import org.junit.runner.RunWith @@ -47,10 +51,62 @@ class SentryStartupBenchmark { startActivityAndWait() } - private companion object { - const val TARGET_PACKAGE = "io.sentry.samples.android" + // Not private: @AfterClass needs a public static method. + companion object { + private const val TARGET_PACKAGE = "io.sentry.samples.android" // Matches the android.os.Trace section name in SentryAndroid.init. - const val INIT_TRACE_SECTION = "SentryAndroid.init" + private const val INIT_TRACE_SECTION = "SentryAndroid.init" + + private const val BENCHMARK_DATA_SUFFIX = "-benchmarkData.json" + + /** Kept in sync with `scripts/parse-macrobenchmark-log.py`. */ + private const val LOG_TAG = "SentryBenchmarkData" + + /** + * Well under logcat's ~4 KB per-message cap, so a chunk is never silently truncated, while + * still keeping the whole document to a handful of messages. + */ + private const val CHUNK_LENGTH = 2000 + + /** + * Echoes the benchmark results into logcat so CI can recover them. + * + * Macrobenchmark reports its numbers two ways, and on Sauce Labs neither one arrives: the + * human-readable summary goes into the instrumentation status bundle (which only Studio and AGP + * read), and `-benchmarkData.json` is written to the app's external media dir, which Sauce + * cannot pull — it only returns assets it produces itself, and logcat is one of them. + * + * Safe to run at this point because `ResultWriter` writes the file synchronously as each result + * is appended; only its *reporting* is deferred to the end of the run. Locally this is just + * extra logcat noise — Gradle still copies the real file into the build directory. + */ + @JvmStatic + @AfterClass + fun logBenchmarkDataToLogcat() { + val benchmarkData = findBenchmarkData() + if (benchmarkData == null) { + Log.w(LOG_TAG, "No *$BENCHMARK_DATA_SUFFIX found, cannot report results to CI") + return + } + + // Dropping the indentation makes the JSON compact enough to survive as a few logcat + // messages. Only structural whitespace is affected: JSON forbids raw newlines inside + // strings, so no value can span lines or start with the indentation being trimmed. + val compactJson = benchmarkData.readText().lineSequence().joinToString("") { it.trimStart() } + val chunks = compactJson.chunked(CHUNK_LENGTH) + chunks.forEachIndexed { index, chunk -> + Log.i(LOG_TAG, "[${index + 1}/${chunks.size}]$chunk") + } + } + + private fun findBenchmarkData(): File? { + val context = InstrumentationRegistry.getInstrumentation().targetContext + @Suppress("DEPRECATION") + val candidateDirs = context.externalMediaDirs.toList() + context.externalCacheDir + return candidateDirs.filterNotNull().firstNotNullOfOrNull { dir -> + dir.listFiles()?.firstOrNull { it.name.endsWith(BENCHMARK_DATA_SUFFIX) } + } + } } } From 2972f5a8aafee8898502ef9b62d20d293242ba52 Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Thu, 6 Aug 2026 18:15:52 +0200 Subject: [PATCH 09/15] ci(macrobenchmark): Decode Sauce's JSON-lines device log The recovery worked on the device but not in the parser. Sauce returns device.log as JSON lines -- {"tag", "message", "level", ...} -- so the payload arrives with its quotes escaped and cannot be regexed straight out of the raw line. Decode each entry and match against its message, falling back to the raw line so the same parser still handles `adb logcat` output from a local run. Enable pipefail in the workflow step too. Piping into tee meant the step exited on tee's status, so this very failure reported success while producing no results. Verified against the device.log from run 31117974318: timeToInitialDisplayMs min 443.5 / median 477.3 / max 571.2 over 12 iterations on a Pixel 9 Pro XL. --- .../integration-tests-macrobenchmark.yml | 3 +++ scripts/parse-macrobenchmark-log.py | 25 ++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests-macrobenchmark.yml b/.github/workflows/integration-tests-macrobenchmark.yml index f75afdbc73e..64e25af8a01 100644 --- a/.github/workflows/integration-tests-macrobenchmark.yml +++ b/.github/workflows/integration-tests-macrobenchmark.yml @@ -70,6 +70,9 @@ jobs: - name: Recover benchmark results from the device log if: always() && env.SAUCE_USERNAME != null run: | + # Without pipefail the step passes on `tee`'s exit code, so a failed recovery + # would report success while silently producing no results. + set -o pipefail python3 scripts/parse-macrobenchmark-log.py ./artifacts \ --json-out ./artifacts/benchmarkData.json | tee -a "$GITHUB_STEP_SUMMARY" diff --git a/scripts/parse-macrobenchmark-log.py b/scripts/parse-macrobenchmark-log.py index 3fad2122b9e..e1b13e1dcac 100755 --- a/scripts/parse-macrobenchmark-log.py +++ b/scripts/parse-macrobenchmark-log.py @@ -19,14 +19,33 @@ CHUNK_RE = re.compile(r"SentryBenchmarkData\s*:\s*\[(\d+)/(\d+)\](.*)$") +def log_messages(log_file): + """Yields the message text of every log entry. + + Sauce hands back device.log as JSON lines -- {"tag", "message", "level", ...} -- which + means the payload arrives with its quotes escaped, so it has to be decoded rather than + regexed out of the raw line. Plain-text lines are passed through unchanged so the same + parser works on `adb logcat` output from a local run. + """ + # Sauce device logs occasionally carry undecodable bytes; don't die on them. + for line in log_file.read_text(errors="replace").splitlines(): + line = line.strip() + if line.startswith("{"): + try: + yield json.loads(line).get("message", "") + continue + except json.JSONDecodeError: + pass + yield line + + def collect_chunks(log_files): """Returns the chunk texts keyed by index, plus the expected total.""" chunks = {} total = None for log_file in log_files: - # Sauce device logs occasionally carry undecodable bytes; don't die on them. - for line in log_file.read_text(errors="replace").splitlines(): - match = CHUNK_RE.search(line) + for message in log_messages(log_file): + match = CHUNK_RE.search(message) if not match: continue index, chunk_total, text = int(match.group(1)), int(match.group(2)), match.group(3) From c212da553835e3262cae844dde978a9774519a55 Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Thu, 6 Aug 2026 18:44:53 +0200 Subject: [PATCH 10/15] docs(macrobenchmark): Record how Sauce results are retrieved Document the three routes that do not work -- artifacts.download.match, Macrobenchmark's own reporting channels, and the Real Device Access API on a public-cloud account -- so the next person does not re-derive them. --- .../README.md | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/README.md b/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/README.md index 4b89b7b6105..9914825584d 100644 --- a/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/README.md +++ b/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/README.md @@ -34,6 +34,34 @@ Connect a device, then: Results print to the console and are written to `build/outputs/connected_android_test_additional_output/.../*-benchmarkData.json`. +## Running on Sauce Labs + +The `Integration Tests - Macrobenchmark` workflow (manual trigger) runs the same benchmark on a +Sauce Labs real device. It reports numbers only — it is not a PR gate, because cloud devices run +with unlocked CPU clocks and the run-to-run spread swamps most SDK-init changes. + +Getting the numbers *back off* the device is the awkward part, so if you are changing this, know +what has already been ruled out: + +- **`artifacts.download.match` in `.sauce/*.yml` cannot reach the device.** It filters a + hardcoded list of assets Sauce hosts for the job (`device.log`, `junit.xml`, `video.mp4`, + `network.har`, `crash.json`, `screenshots.zip`); nothing enumerates device storage. A + `*-benchmarkData.json` pattern there matches nothing. +- **Macrobenchmark's own reporting channels don't survive.** The readable summary goes into the + instrumentation status bundle, which only Studio and AGP consume, and `benchmarkData.json` is + written to the app's external media dir, which Sauce never pulls. +- **The Real Device Access API can pull device files, but not on our account.** It offers + `pullFile` and `executeShellCommand`, and `GET /rdc/v2/devices/status` even lists the whole + public fleet — but `POST /sessions` answers `deviceClasses=[PRIVATE_DEVICE]` and there is no + parameter to request a public device. It would need leased private devices. That route would + also return the per-iteration perfetto traces, so it is worth revisiting if we ever get them. + +So `SentryStartupBenchmark` echoes its own `benchmarkData.json` into logcat in chunks, which +reaches CI inside `device.log`, and `scripts/parse-macrobenchmark-log.py` reassembles it and +writes a `timeToInitialDisplay` table to the job summary. Note this recovers the metrics only — +the perfetto traces are megabytes each and cannot go through logcat, so sub-millisecond work +still needs a local device. + ### Device hygiene (do this for trustworthy numbers) - **Wake and unlock the device first** — the launch check fails with "Unable to confirm activity From f23948c0c6c6018dc841b8f745ef8da92e7c132b Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Fri, 7 Aug 2026 09:48:10 +0200 Subject: [PATCH 11/15] refactor(macrobenchmark): Collect log chunks per file Chunks were merged across every *.log found, so two devices each emitting the same number of chunks would overwrite each other's indices and reassemble into one bogus document. Collect per file and report when more than one log carries results, which also drops the cross-file total comparison that was standing in for this check. Build the summary table and per-iteration details in one pass while here. --- scripts/parse-macrobenchmark-log.py | 72 ++++++++++++++++------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/scripts/parse-macrobenchmark-log.py b/scripts/parse-macrobenchmark-log.py index e1b13e1dcac..280970dbea8 100755 --- a/scripts/parse-macrobenchmark-log.py +++ b/scripts/parse-macrobenchmark-log.py @@ -39,23 +39,14 @@ def log_messages(log_file): yield line -def collect_chunks(log_files): - """Returns the chunk texts keyed by index, plus the expected total.""" - chunks = {} - total = None - for log_file in log_files: - for message in log_messages(log_file): - match = CHUNK_RE.search(message) - if not match: - continue - index, chunk_total, text = int(match.group(1)), int(match.group(2)), match.group(3) - if total is not None and chunk_total != total: - sys.exit( - f"Found chunks from more than one benchmark run " - f"(totals {total} and {chunk_total}) in {log_file}" - ) - total = chunk_total - chunks[index] = text +def collect_chunks(log_file): + """Returns one log's chunk texts keyed by index, plus the expected total.""" + chunks, total = {}, None + for message in log_messages(log_file): + match = CHUNK_RE.search(message) + if match: + index, total = int(match.group(1)), int(match.group(2)) + chunks[index] = match.group(3) return chunks, total @@ -86,25 +77,30 @@ def format_summary(data): "", ] - lines += ["| Benchmark | Metric | min | median | max | CoV | iterations |", "|---|---|--:|--:|--:|--:|--:|"] + table = [ + "| Benchmark | Metric | min | median | max | CoV | iterations |", + "|---|---|--:|--:|--:|--:|--:|", + ] + details = [] for benchmark in data["benchmarks"]: + name = f"{benchmark['className'].rsplit('.', 1)[-1]}.{benchmark['name']}" for metric, result in sorted(benchmark["metrics"].items()): - lines.append( - f"| `{benchmark['className'].rsplit('.', 1)[-1]}.{benchmark['name']}` " - f"| {metric} " - f"| {result['minimum']:.1f} " - f"| {result['median']:.1f} " - f"| {result['maximum']:.1f} " - f"| {result['coefficientOfVariation'] * 100:.1f}% " - f"| {len(result['runs'])} |" + table.append( + f"| `{name}` | {metric} " + f"| {result['minimum']:.1f} | {result['median']:.1f} | {result['maximum']:.1f} " + f"| {result['coefficientOfVariation'] * 100:.1f}% | {len(result['runs'])} |" ) - - for benchmark in data["benchmarks"]: - for metric, result in sorted(benchmark["metrics"].items()): runs = ", ".join(f"{run:.1f}" for run in result["runs"]) - lines += ["", f"
{metric} per iteration", "", runs, "", "
"] + details += [ + "", + f"
{metric} per iteration", + "", + runs, + "", + "
", + ] - return "\n".join(lines) + return "\n".join(lines + table + details) def main(): @@ -117,12 +113,22 @@ def main(): if not log_files: sys.exit(f"No *.log files under {args.artifacts_dir}") - chunks, total = collect_chunks(log_files) - if total is None: + # Keyed by file so chunks from two devices can never be merged into one bogus document. + per_log = {log: collect_chunks(log) for log in log_files} + with_chunks = {log: result for log, (result, total) in per_log.items() if total} + if not with_chunks: sys.exit( "No SentryBenchmarkData chunks in the device log. The benchmark most likely " "failed before reporting — check junit.xml and the log for Macrobenchmark errors." ) + if len(with_chunks) > 1: + sys.exit( + "Chunks from more than one run: " + + ", ".join(str(log) for log in with_chunks) + + ". This parser reports a single device." + ) + log_file = next(iter(with_chunks)) + chunks, total = per_log[log_file] data = json.loads(reassemble(chunks, total)) From edf75b39dba64e3bd072fa41cf10ffb05a7bdeea Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Fri, 7 Aug 2026 10:22:39 +0200 Subject: [PATCH 12/15] ci(macrobenchmark): Drop unused submodule checkout and encryption key env The repo has no submodules -- no .gitmodules, no gitlink entries -- so submodules: 'recursive' does nothing; sentry-native-ndk is a Maven dependency pinned in gradle/libs.versions.toml. GRADLE_ENCRYPTION_KEY is never read as env.* or $GRADLE_ENCRYPTION_KEY either; setup-gradle takes the secret directly via cache-encryption-key, which stays. --- .github/workflows/integration-tests-macrobenchmark.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/integration-tests-macrobenchmark.yml b/.github/workflows/integration-tests-macrobenchmark.yml index 64e25af8a01..d2ec259e2c1 100644 --- a/.github/workflows/integration-tests-macrobenchmark.yml +++ b/.github/workflows/integration-tests-macrobenchmark.yml @@ -32,13 +32,10 @@ jobs: # we copy the secret to the env variable in order to access it in the workflow env: SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} - GRADLE_ENCRYPTION_KEY: ${{ secrets.GRADLE_ENCRYPTION_KEY }} steps: - name: Git checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - submodules: 'recursive' - name: 'Set up Java: 17' uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5 From c1cd5bda9175b93739379ccc897b5cdc817dab46 Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Fri, 7 Aug 2026 14:51:08 +0200 Subject: [PATCH 13/15] ci(macrobenchmark): Record why the benchmark runs on a high-end device Older hardware would resolve SDK-init regressions better, but Google_Pixel_4_13_real_us could not be allocated in three attempts -- each queued ~61 minutes before Sauce gave up with a concurrency retry-timeout, leaving an empty device log and tests="0". The Pixel 9 Pro XL allocates reliably and is the device this pipeline is proven green on. --- .sauce/sentry-uitest-android-macrobenchmark.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.sauce/sentry-uitest-android-macrobenchmark.yml b/.sauce/sentry-uitest-android-macrobenchmark.yml index 4b45388c8d9..46f2e4ba6f7 100644 --- a/.sauce/sentry-uitest-android-macrobenchmark.yml +++ b/.sauce/sentry-uitest-android-macrobenchmark.yml @@ -26,6 +26,14 @@ suites: # No test orchestrator and no clearPackageData: Macrobenchmark manages its own process # restarts and AOT compilation, and StartupMode.COLD intentionally keeps app data and # permissions (it force-stops rather than `pm clear`). + # + # Older hardware would give a better signal -- it spends longer in SentryAndroid.init, so a + # regression is larger against the same relative noise -- but Google_Pixel_4_13_real_us could + # not be allocated in three attempts, each queueing ~61 minutes before Sauce gave up with a + # concurrency retry-timeout. This device allocates reliably. + # + # Keep this to one device: two would produce two device logs, which the parser rejects rather + # than splice together -- see scripts/parse-macrobenchmark-log.py. devices: - id: Google_Pixel_9_Pro_XL_15_real_sjc1 # Google Pixel 9 Pro XL - api 35 (15) - high end From e4b8ddf16e17f660bbe33b46db70770b4c1f4aa5 Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Fri, 7 Aug 2026 15:23:21 +0200 Subject: [PATCH 14/15] ref(macrobenchmark): Narrow the deprecation suppression to the deprecated call @Suppress("DEPRECATION") covered the whole candidateDirs expression, including externalCacheDir, which is not deprecated -- so a future deprecation there would have been silently swallowed. Scope it to getExternalMediaDirs and record why it cannot be migrated: it is deprecated in favour of MediaStore, which returns content URIs rather than the filesystem path androidx.benchmark writes to. --- .../android/macrobenchmark/SentryStartupBenchmark.kt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt b/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt index 6099b45fa3d..87669a79205 100644 --- a/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt +++ b/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt @@ -102,9 +102,12 @@ class SentryStartupBenchmark { private fun findBenchmarkData(): File? { val context = InstrumentationRegistry.getInstrumentation().targetContext - @Suppress("DEPRECATION") - val candidateDirs = context.externalMediaDirs.toList() + context.externalCacheDir - return candidateDirs.filterNotNull().firstNotNullOfOrNull { dir -> + // Deprecated since API 30 in favour of MediaStore, which hands back content URIs rather + // than the filesystem path androidx.benchmark writes its File to -- so there is nothing to + // migrate to. Suppressed on this call alone; externalCacheDir below is not deprecated. + @Suppress("DEPRECATION") val mediaDirs = context.externalMediaDirs.toList() + // Outputs uses the media dir from API 29 on, and externalCacheDir on API 24-28. + return (mediaDirs + context.externalCacheDir).filterNotNull().firstNotNullOfOrNull { dir -> dir.listFiles()?.firstOrNull { it.name.endsWith(BENCHMARK_DATA_SUFFIX) } } } From fdded9e044019e33d37e366ff81e3cc675e0090d Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Fri, 7 Aug 2026 16:13:10 +0200 Subject: [PATCH 15/15] ref(macrobenchmark): Trim comments and drop the cache encryption key The retrieval rationale these comments carried is documented in the module README, so removing the duplicated prose loses nothing. Also drops cache-encryption-key from the Setup Gradle step, leaving this workflow without the Gradle configuration-cache encryption the other workflows still configure. --- .../integration-tests-macrobenchmark.yml | 11 ----------- .sauce/sentry-uitest-android-macrobenchmark.yml | 15 --------------- .../macrobenchmark/SentryStartupBenchmark.kt | 14 ++------------ 3 files changed, 2 insertions(+), 38 deletions(-) diff --git a/.github/workflows/integration-tests-macrobenchmark.yml b/.github/workflows/integration-tests-macrobenchmark.yml index d2ec259e2c1..0ce486d83df 100644 --- a/.github/workflows/integration-tests-macrobenchmark.yml +++ b/.github/workflows/integration-tests-macrobenchmark.yml @@ -2,15 +2,6 @@ name: 'Integration Tests - Macrobenchmark' # Runs the sentry-uitest-android-macrobenchmark cold-start benchmark on a Sauce Labs real # device and recovers timeToInitialDisplay from the device log. # -# Getting the numbers back is the awkward part. saucectl only downloads assets Sauce hosts -# for a job (device.log, junit.xml, video.mp4, ...) and never reads the device filesystem, -# so Macrobenchmark's benchmarkData.json cannot be fetched directly. The Real Device Access -# API can pull device files, but POST /sessions accepts only private devices -# (deviceClasses=[PRIVATE_DEVICE]) and we run on the public cloud. So the benchmark echoes -# its own results into logcat, and scripts/parse-macrobenchmark-log.py reassembles them. -# -# Manual trigger only -- this reports numbers, it does not gate PRs. Cloud devices have -# unlocked CPU clocks, so run-to-run spread is far wider than most SDK-init changes. on: workflow_dispatch: # Temporary scaffolding: workflow_dispatch cannot target a workflow that does not exist on @@ -45,8 +36,6 @@ jobs: - name: Setup Gradle uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.3.0 - with: - cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - name: Assemble target app and Macrobenchmark apk if: env.SAUCE_USERNAME != null diff --git a/.sauce/sentry-uitest-android-macrobenchmark.yml b/.sauce/sentry-uitest-android-macrobenchmark.yml index 46f2e4ba6f7..4e3d5cfbb3d 100644 --- a/.sauce/sentry-uitest-android-macrobenchmark.yml +++ b/.sauce/sentry-uitest-android-macrobenchmark.yml @@ -14,10 +14,7 @@ defaults: timeout: 40m espresso: - # Target app under test: Macrobenchmark cold-starts sentry-samples-android. It must be - # release-like; the release build type is signed with the debug key so it installs on Sauce. app: ./sentry-samples/sentry-samples-android/build/outputs/apk/release/sentry-samples-android-release.apk - # Instrumentation APK: the self-instrumenting com.android.test macrobenchmark module. testApp: ./sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/build/outputs/apk/benchmark/sentry-uitest-android-macrobenchmark-benchmark.apk suites: @@ -26,21 +23,9 @@ suites: # No test orchestrator and no clearPackageData: Macrobenchmark manages its own process # restarts and AOT compilation, and StartupMode.COLD intentionally keeps app data and # permissions (it force-stops rather than `pm clear`). - # - # Older hardware would give a better signal -- it spends longer in SentryAndroid.init, so a - # regression is larger against the same relative noise -- but Google_Pixel_4_13_real_us could - # not be allocated in three attempts, each queueing ~61 minutes before Sauce gave up with a - # concurrency retry-timeout. This device allocates reliably. - # - # Keep this to one device: two would produce two device logs, which the parser rejects rather - # than splice together -- see scripts/parse-macrobenchmark-log.py. devices: - id: Google_Pixel_9_Pro_XL_15_real_sjc1 # Google Pixel 9 Pro XL - api 35 (15) - high end -# The device log carries the actual results: Sauce only returns assets it produces itself, so it -# cannot pull Macrobenchmark's `-benchmarkData.json` off the device. SentryStartupBenchmark -# echoes that JSON into logcat instead, and scripts/parse-macrobenchmark-log.py reassembles it. -# The log also holds Macrobenchmark's device guard warnings (unlocked clocks, low battery, ...). artifacts: download: when: always diff --git a/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt b/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt index 87669a79205..824a2a0628f 100644 --- a/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt +++ b/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt @@ -27,10 +27,6 @@ import org.junit.runner.RunWith * [android.os.Trace] section the SDK emits, isolating SDK-init cost from the rest of the start. * * [CompilationMode.Full] pins ART AOT compilation so dexopt state does not drift between runs. - * Iterations are capped at 12: on an unthrottled Pixel 3, back-to-back cold starts hit thermal - * throttling after ~14 iterations, which inflates the tail of longer runs. This is NOT a CI test; - * it requires a connected device. To A/B an SDK change, see README.md (build the app twice, once - * per SDK variant, in interleaved rounds). */ @OptIn(ExperimentalMetricApi::class) @RunWith(AndroidJUnit4::class) @@ -63,10 +59,7 @@ class SentryStartupBenchmark { /** Kept in sync with `scripts/parse-macrobenchmark-log.py`. */ private const val LOG_TAG = "SentryBenchmarkData" - /** - * Well under logcat's ~4 KB per-message cap, so a chunk is never silently truncated, while - * still keeping the whole document to a handful of messages. - */ + /** Well under logcat's ~4 KB per-message cap, so a chunk is never silently truncated. */ private const val CHUNK_LENGTH = 2000 /** @@ -102,11 +95,8 @@ class SentryStartupBenchmark { private fun findBenchmarkData(): File? { val context = InstrumentationRegistry.getInstrumentation().targetContext - // Deprecated since API 30 in favour of MediaStore, which hands back content URIs rather - // than the filesystem path androidx.benchmark writes its File to -- so there is nothing to - // migrate to. Suppressed on this call alone; externalCacheDir below is not deprecated. + // This is where Macrobenchmark writes to for some reason. @Suppress("DEPRECATION") val mediaDirs = context.externalMediaDirs.toList() - // Outputs uses the media dir from API 29 on, and externalCacheDir on API 24-28. return (mediaDirs + context.externalCacheDir).filterNotNull().firstNotNullOfOrNull { dir -> dir.listFiles()?.firstOrNull { it.name.endsWith(BENCHMARK_DATA_SUFFIX) } }