This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Paywall preload benchmark — opt-in only. Runs on PRs into main or develop | |
| # that carry the `release` or `benchmark` label, or on manual dispatch. | |
| # | |
| # Measures the time from Superwall.preloadAllPaywalls() until every paywall for | |
| # the dev app's embedded API key reaches PaywallLoadingState.Ready, on 3 | |
| # emulators in parallel (LOW / MID / HIGH device tiers). Every tier runs the | |
| # SAME pinned device image (pixel_6, API 34, google_apis_playstore, x86_64 — | |
| # Play Store enabled); only CPU cores and RAM differ, so results are comparable | |
| # run-to-run. See .benchmark/README.md. | |
| # | |
| # Emulator setup (KVM, free-disk, AVD snapshot cache, -no-snapshot-save on the | |
| # measured run) mirrors pr-tests.yml / build+test.yml, which are proven on | |
| # these runners; the Play Store device choice (pixel_6) matches the Google | |
| # Play testing done via DeviceCloud in maestro-run.yml. | |
| # | |
| # The report is written to .benchmark/ (committed back to the PR branch) and | |
| # the workflow fails when a tier's mean regresses more than the delta limit in | |
| # .benchmark/config.json (default 10%). | |
| name: Paywall Preload Benchmark | |
| on: | |
| # PRs into main (release) and develop (feature) run the benchmark only when | |
| # opted in with the `release` or `benchmark` label — it burns 3 emulators for | |
| # up to 40 minutes, so it's not worth paying on every PR push. `labeled` is in | |
| # the trigger list so adding the label to an open PR starts the run. | |
| pull_request: | |
| branches: [ main, develop ] | |
| types: [ opened, synchronize, reopened, labeled ] | |
| # No push trigger: a merge into main/develop carries no labels, so it could | |
| # never satisfy the opt-in. Roll the baseline forward with a manual dispatch | |
| # (update_baseline: true) off the branch you want to become the new mainline. | |
| workflow_dispatch: | |
| inputs: | |
| update_baseline: | |
| description: "Commit this run's results as the new baselines" | |
| type: boolean | |
| default: false | |
| concurrency: | |
| group: preload-benchmark-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| # Build both APKs once; the tier jobs are gradle-free and just install + run. | |
| build-apks: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| # Opt-in guard: PRs need the `release` or `benchmark` label; manual | |
| # dispatches always run. The downstream jobs `needs` this one, so they skip | |
| # with it. | |
| if: >- | |
| github.event_name == 'workflow_dispatch' || | |
| contains(github.event.pull_request.labels.*.name, 'release') || | |
| contains(github.event.pull_request.labels.*.name, 'benchmark') | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '17' | |
| distribution: 'temurin' | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@v3 | |
| - name: Change wrapper permissions | |
| run: chmod +x ./gradlew | |
| # benchmarkTestMode bakes BuildConfig.BENCHMARK_TEST_MODE=true into the | |
| # APKs: the benchmark then runs the SDK with TestModeBehavior.ALWAYS so | |
| # BILLING_UNAVAILABLE (no Play account on CI emulators) doesn't fail | |
| # paywalls with products. | |
| - name: Assemble app + androidTest APKs | |
| run: ./gradlew :app:assembleDebug :app:assembleDebugAndroidTest -PbenchmarkTestMode=true -x lint -x lintDebug -x lintVitalRelease | |
| - name: Upload APKs | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-apks | |
| path: | | |
| app/build/outputs/apk/debug/app-debug.apk | |
| app/build/outputs/apk/androidTest/debug/app-debug-androidTest.apk | |
| if-no-files-found: error | |
| retention-days: 1 | |
| benchmark: | |
| runs-on: ubuntu-latest | |
| needs: [build-apks] | |
| # Hard cap — a hung emulator/test must fail the job, not ride the runner | |
| # for GitHub's 6h default. Sized for 10 runs x 20 paywalls per tier. | |
| timeout-minutes: 40 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # Device tiers: identical image, different horsepower. RAM follows the | |
| # SDK's own DeviceClassifier tier definitions (LOW = 3-4GB; 2GB would | |
| # model ULTRA_LOW and OOM-kills webview-heavy benchmarks). | |
| - tier: low | |
| cores: 2 | |
| ram: 3072M | |
| - tier: mid | |
| cores: 3 | |
| ram: 4096M | |
| - tier: high | |
| cores: 4 | |
| ram: 6144M | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| # Play Store images are large; reclaim space before the AVD restore + boot. | |
| - name: Free disk space | |
| uses: jlumbroso/free-disk-space@v1.3.1 | |
| with: | |
| android: false | |
| tool-cache: true | |
| dotnet: true | |
| haskell: true | |
| swap-storage: true | |
| docker-images: false | |
| large-packages: false | |
| - name: Enable KVM | |
| run: | | |
| echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules | |
| sudo udevadm control --reload-rules | |
| sudo udevadm trigger --name-match=kvm | |
| - name: Read benchmark config | |
| id: config | |
| run: | | |
| echo "runs=$(jq -r '.runsPerEmulator' .benchmark/config.json)" >> "$GITHUB_OUTPUT" | |
| echo "iterations=$(jq -r '.iterations' .benchmark/config.json)" >> "$GITHUB_OUTPUT" | |
| echo "timeout=$(jq -r '.timeoutSec' .benchmark/config.json)" >> "$GITHUB_OUTPUT" | |
| echo "placements=$(jq -r '(.placements // []) | join(",")' .benchmark/config.json)" >> "$GITHUB_OUTPUT" | |
| - name: Download prebuilt APKs | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchmark-apks | |
| path: apks/ | |
| # Cache the AVD per tier. The key pins the exact device the benchmark runs | |
| # on — bump the -v suffix to force a fresh image. | |
| - name: AVD cache | |
| uses: actions/cache@v4 | |
| id: avd-cache | |
| with: | |
| path: | | |
| ~/.android/avd/* | |
| ~/.android/adb* | |
| # v3: low tier RAM changed (snapshots don't survive RAM config changes). | |
| key: avd-benchmark-api34-playstore-x86_64-pixel6-${{ matrix.tier }}-v3 | |
| # Cold-cache only: create the AVD and let the emulator save a snapshot on shutdown. | |
| - name: Create AVD and generate snapshot for caching | |
| if: steps.avd-cache.outputs.cache-hit != 'true' | |
| uses: reactivecircus/android-emulator-runner@v2 | |
| with: | |
| api-level: 34 | |
| target: playstore | |
| arch: x86_64 | |
| profile: pixel_6 | |
| cores: ${{ matrix.cores }} | |
| ram-size: ${{ matrix.ram }} | |
| force-avd-creation: false | |
| emulator-boot-timeout: 1800 | |
| disable-animations: false | |
| # -dns-server: GitHub runners resolve via systemd-resolved (127.0.0.53), | |
| # which the emulator guest can't use — without an explicit DNS server the | |
| # guest cannot resolve any hostname and the SDK's config fetch never | |
| # completes. | |
| emulator-options: > | |
| -dns-server 8.8.8.8 | |
| -no-window | |
| -gpu swiftshader_indirect | |
| -no-boot-anim | |
| -noaudio | |
| -camera-back none | |
| -camera-front none | |
| script: echo "Generated AVD snapshot for caching." | |
| - name: Run preload benchmark (${{ matrix.tier }} tier) | |
| uses: reactivecircus/android-emulator-runner@v2 | |
| with: | |
| api-level: 34 | |
| target: playstore | |
| arch: x86_64 | |
| profile: pixel_6 | |
| cores: ${{ matrix.cores }} | |
| ram-size: ${{ matrix.ram }} | |
| force-avd-creation: false | |
| emulator-boot-timeout: 1800 | |
| disable-animations: true | |
| # -dns-server: see the snapshot step above — required for the guest to | |
| # resolve api.superwall.me on GitHub runners. | |
| emulator-options: > | |
| -dns-server 8.8.8.8 | |
| -no-snapshot-save | |
| -no-window | |
| -gpu swiftshader_indirect | |
| -no-boot-anim | |
| -noaudio | |
| -camera-back none | |
| -camera-front none | |
| # android-emulator-runner executes each script line in its own `sh -c`, | |
| # so all logic (5s post-boot settle, install, RUNS instrument invocations, | |
| # result pull) lives in a single-line script call. The script installs the | |
| # prebuilt APKs once, then per run does `pm clear` + `am instrument` — so | |
| # every run starts cold with zero gradle overhead; the compare script | |
| # averages across all runs. | |
| script: bash scripts/run_preload_benchmark.sh ${{ matrix.tier }} ${{ steps.config.outputs.runs }} ${{ steps.config.outputs.iterations }} ${{ steps.config.outputs.timeout }} apks/debug/app-debug.apk apks/androidTest/debug/app-debug-androidTest.apk "${{ steps.config.outputs.placements }}" | |
| - name: Upload benchmark results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: preload-benchmark-${{ matrix.tier }} | |
| path: app/build/outputs/benchmark/ | |
| # On a failed run there are legitimately no results — don't stack a | |
| # confusing artifact error on top; the report job fails on missing tiers. | |
| if-no-files-found: warn | |
| report: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| needs: [benchmark] | |
| # Produce a report from whatever tiers finished, even if one regressed/failed | |
| # — but not when the benchmark job was skipped by the opt-in guard. | |
| if: ${{ !cancelled() && needs.benchmark.result != 'skipped' }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Download benchmark results (all tiers) | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: preload-benchmark-* | |
| path: benchmark-artifacts/ | |
| - name: Compare against baselines and build report | |
| id: compare | |
| run: | | |
| set +e | |
| python3 scripts/benchmark_compare.py \ | |
| --results benchmark-artifacts \ | |
| --baseline .benchmark/baseline \ | |
| --config .benchmark/config.json \ | |
| --report .benchmark/REPORT.md \ | |
| --normalized-out .benchmark/results \ | |
| ${{ (github.event_name == 'workflow_dispatch' && inputs.update_baseline) && '--update-baseline' || '' }} | |
| echo "exit_code=$?" >> "$GITHUB_OUTPUT" | |
| - name: Publish report to job summary | |
| if: always() | |
| run: cat .benchmark/REPORT.md >> "$GITHUB_STEP_SUMMARY" || true | |
| - name: Commit report to branch | |
| # PRs from forks can't be pushed to; pushes/dispatch always target our own ref. | |
| if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository | |
| run: | | |
| git config --global user.name 'github-actions[bot]' | |
| git config --global user.email 'github-actions[bot]@users.noreply.github.com' | |
| git add .benchmark/ | |
| if git diff --staged --quiet; then | |
| echo "No benchmark changes to commit" | |
| else | |
| git commit -m "Update paywall preload benchmark report [skip ci]" | |
| git push origin HEAD:${{ github.head_ref || github.ref_name }} | |
| fi | |
| - name: Fail on regression over delta limit | |
| run: | | |
| code="${{ steps.compare.outputs.exit_code }}" | |
| if [ "$code" != "0" ]; then | |
| echo "Benchmark comparison failed with exit code $code (3 = regression over delta limit, 4 = missing results)." | |
| exit "$code" | |
| fi |