name: Perf Nightly

# Nightly iOS and Android command performance against the fixture app, compared night over night

# per platform (#1781 A3).

# - Catches: a command whose daemon time moves beyond the noise of earlier nights, or that starts

# failing. PR lanes check behavior, not time.

# - Wanted for: tuning runner, snapshot, and settle paths while watching the effect. iOS medians

# swing up to +122% between single nights, so a step is judged against the median of up to seven

# earlier nights and must clear their spread, its own sampling error, 20%, and 50 ms. A night

# runs 3 rounds, the minimum for a timing verdict, so one failed sample withholds that step's

# timing verdict.

# - Cost: one macos-26 job of about 45 minutes and one ubuntu emulator job a night; summaries are

# kept 90 days, raw logs 30.

# - Delete or park when: no flagged change has been acted on for 60 nights.

on:

schedule:

- cron: '40 2 * * *'

workflow_dispatch:

inputs:

rounds:

description: 'Measured rounds per step (samples, at least 3)'

required: false

default: '3'

warmup:

description: 'Warmup rounds dropped before measuring'

required: false

default: '1'

scenarios:

description: 'Comma-separated scenario ids (empty runs all)'

required: false

default: ''

baseline-nights:

description: 'Earlier scheduled nights in the rolling baseline'

required: false

default: '7'

baseline-branch:

description: 'Branch whose runs form the baseline'

required: false

default: 'main'

ios:

description: 'Run the iOS suite'

type: boolean

required: false

default: true

android:

description: 'Run the Android suite'

type: boolean

required: false

default: true

permissions:

contents: read

actions: read

concurrency:

group: perf-nightly-${{ github.ref }}

cancel-in-progress: false

env:

PERF_ROUNDS: ${{ github.event.inputs.rounds || '3' }}

PERF_WARMUP: ${{ github.event.inputs.warmup || '1' }}

jobs:

perf-ios:

name: iOS Command Perf

if: ${{ github.event_name == 'schedule' || github.event.inputs.ios == 'true' }}

runs-on: blacksmith-6vcpu-macos-26

timeout-minutes: 100

env:

IOS_RUNTIME_VERSION: '26.2'

AGENT_DEVICE_XCUITEST_ARCHS: 'arm64'

AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state

AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived

AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS: '420000'

steps:

- name: Checkout

uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

- name: Setup toolchain

uses: ./.github/actions/setup-node-pnpm

with:

cache-dependency-path: |

pnpm-lock.yaml

examples/test-app/pnpm-lock.yaml

- name: Build the shared CLI

uses: ./.github/actions/run-gate

with:

gate: build

- name: Boot iOS test simulator

id: ios-simulator

uses: ./.github/actions/boot-ios-test-simulator

with:

runtime-version: ${{ env.IOS_RUNTIME_VERSION }}

preferred-device-name: iPhone 17 Pro

- name: Restore and build iOS XCTest runner

uses: ./.github/actions/setup-apple-runner-build

with:

derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }}

cache-key-prefix: ios-runner-prebuilt

cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}-arm64

gate: swift-runner-ios

xcuitest-platform: ios

xcuitest-destination: generic/platform=iOS Simulator

- name: Install the fixture app

id: fixture-app

uses: ./.github/actions/setup-fixture-app

- name: Preflight iOS runner through public CLI

env:

UDID: ${{ steps.ios-simulator.outputs.simulator-udid }}

run: |

# The keyboard must stay expanded for keyboard dismiss to find its Done key.

xcrun simctl spawn "$UDID" defaults write com.apple.keyboard.preferences AutomaticMinimizationEnabled -bool false

pnpm clean:daemon

node bin/agent-device.mjs prepare ios-runner --platform ios --udid "$UDID" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS"

pnpm clean:daemon

- name: Run iOS command perf suite

env:

UDID: ${{ steps.ios-simulator.outputs.simulator-udid }}

APP_ID: ${{ steps.fixture-app.outputs.app-id }}

APP_PATH: ${{ steps.fixture-app.outputs.app-path }}

SCENARIOS: ${{ github.event.inputs.scenarios }}

run: |

pnpm perf --platform ios --udid "$UDID" \

--app-id "$APP_ID" --app-path "$APP_PATH" \

--rounds "$PERF_ROUNDS" --warmup "$PERF_WARMUP" \

${SCENARIOS:+--scenarios "$SCENARIOS"} \

--out-dir "$GITHUB_WORKSPACE/perf-results"

- name: Compare with earlier nights

if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }}

env:

GH_TOKEN: ${{ github.token }}

BASELINE_BRANCH: ${{ github.event.inputs.baseline-branch || 'main' }}

BASELINE_NIGHTS: ${{ github.event.inputs.baseline-nights || '7' }}

run: |

pnpm perf:compare --platform ios --to perf-results \

--branch "$BASELINE_BRANCH" \

--nights "$BASELINE_NIGHTS" \

--exclude-run "$GITHUB_RUN_ID" \

--out perf-results --github

echo "Full report: the perf-ios-summary artifact holds report.md and summary.json." >> "$GITHUB_STEP_SUMMARY"

- name: Upload iOS perf summary

if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }}

uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2

with:

name: perf-ios-summary

path: |

perf-results/summary.json

perf-results/report.md

perf-results/comparison.json

perf-results/comparison.md

retention-days: 90

if-no-files-found: error

- name: Upload iOS perf raw logs

if: ${{ always() }}

uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2

with:

name: perf-ios-raw

path: perf-results/raw/

retention-days: 30

if-no-files-found: warn

perf-android:

name: Android Command Perf

if: ${{ github.event_name == 'schedule' || github.event.inputs.android == 'true' }}

runs-on: blacksmith-4vcpu-ubuntu-2404

timeout-minutes: 100

steps:

- name: Checkout

uses: useblacksmith/checkout@2b9fe87d796feea153b455ec2f3125b471a7b0e7 # v1.8.2

- name: Setup toolchain

uses: ./.github/actions/setup-node-pnpm

with:

cache-dependency-path: |

pnpm-lock.yaml

examples/test-app/pnpm-lock.yaml

- name: Build the shared CLI

uses: ./.github/actions/run-gate

with:

gate: build

- name: Setup Android replay host

uses: ./.github/actions/setup-android-replay-host

with:

package-helpers: 'true'

- name: Restore the fixture APK

id: fixture-app

uses: ./.github/actions/setup-fixture-app

with:

platform: android

install: 'false'

- name: Setup Android system image

uses: ./.github/actions/setup-android-system-image

# The emulator action runs each script line on its own, so the script is one folded line.

- name: Run Android command perf suite

uses: reactivecircus/android-emulator-runner@b530d96654c385303d652368551fb075bc2f0b6b # v2.35.0

env:

APP_ID: ${{ steps.fixture-app.outputs.app-id }}

APP_PATH: ${{ steps.fixture-app.outputs.apk-path }}

SCENARIOS: ${{ github.event.inputs.scenarios }}

with:

api-level: 36

arch: x86_64

profile: pixel_7

target: google_apis_playstore

emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim -no-metrics

script: >-

set -e;

adb -s emulator-5554 install -r "$APP_PATH";

pnpm perf --platform android --serial emulator-5554

--app-id "$APP_ID" --app-path "$APP_PATH"

--rounds "$PERF_ROUNDS" --warmup "$PERF_WARMUP"

${SCENARIOS:+--scenarios "$SCENARIOS"}

--out-dir "$GITHUB_WORKSPACE/perf-results"

- name: Compare with earlier nights

if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }}

env:

GH_TOKEN: ${{ github.token }}

BASELINE_BRANCH: ${{ github.event.inputs.baseline-branch || 'main' }}

BASELINE_NIGHTS: ${{ github.event.inputs.baseline-nights || '7' }}

run: |

pnpm perf:compare --platform android --to perf-results \

--branch "$BASELINE_BRANCH" \

--nights "$BASELINE_NIGHTS" \

--exclude-run "$GITHUB_RUN_ID" \

--out perf-results --github

echo "Full report: the perf-android-summary artifact holds report.md and summary.json." >> "$GITHUB_STEP_SUMMARY"

- name: Upload Android perf summary

if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }}

uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2

with:

name: perf-android-summary

path: |

perf-results/summary.json

perf-results/report.md

perf-results/comparison.json

perf-results/comparison.md

retention-days: 90

if-no-files-found: error

- name: Upload Android perf raw logs

if: ${{ always() }}

uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2

with:

name: perf-android-raw

path: perf-results/raw/

retention-days: 30

if-no-files-found: warn