Repository navigation
204 lines (183 loc) · 7.73 KB
/
Copy pathperf-nightly.yml
File metadata and controls
204 lines (183 loc) · 7.73 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
name: Perf Nightly
# Nightly iOS command performance against the fixture app, compared night over night (#1781 A3).
# - Catches: a command whose daemon time moves beyond the noise of earlier nights, or that starts
# failing. PR lanes check behavior, not time.
# - Wanted for: tuning runner, snapshot, and settle paths while watching the effect. iOS medians
# swing up to +122% between single nights at n=5, so a step is judged against the median of up to
# seven earlier nights and must clear their spread, its own sampling error, 20%, and 50 ms.
# - Cost: one macos-26 job of about 45 minutes a night; summaries are kept 90 days, raw logs 30.
# - Delete or park when: no flagged change has been acted on for 60 nights.
# Android stays report-only and runs only on request.
on:
schedule:
- cron: '40 2 * * *'
workflow_dispatch:
inputs:
rounds:
description: 'Measured rounds per step (samples, at least 3)'
required: false
default: '3'
warmup:
description: 'Warmup rounds dropped before measuring'
required: false
default: '1'
scenarios:
description: 'Comma-separated iOS scenario ids (empty runs all)'
required: false
default: ''
baseline-nights:
description: 'Earlier scheduled nights in the rolling baseline'
required: false
default: '7'
baseline-branch:
description: 'Branch whose runs form the baseline'
required: false
default: 'main'
android:
description: 'Also run the report-only Android Settings tour'
type: boolean
required: false
default: true
permissions:
contents: read
actions: read
concurrency:
group: perf-nightly-${{ github.ref }}
cancel-in-progress: false
env:
PERF_ROUNDS: ${{ github.event.inputs.rounds || '3' }}
PERF_WARMUP: ${{ github.event.inputs.warmup || '1' }}
jobs:
perf-ios:
name: iOS Command Perf
runs-on: macos-26
timeout-minutes: 100
env:
IOS_RUNTIME_VERSION: '26.2'
AGENT_DEVICE_XCUITEST_ARCHS: 'arm64'
AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state
AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived
AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS: '420000'
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup toolchain
uses: ./.github/actions/setup-node-pnpm
with:
cache-dependency-path: |
pnpm-lock.yaml
examples/test-app/pnpm-lock.yaml
- name: Build the shared CLI
uses: ./.github/actions/run-gate
with:
gate: build
- name: Boot iOS test simulator
id: ios-simulator
uses: ./.github/actions/boot-ios-test-simulator
with:
runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
preferred-device-name: iPhone 17 Pro
- name: Restore and build iOS XCTest runner
uses: ./.github/actions/setup-apple-runner-build
with:
derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }}
cache-key-prefix: ios-runner-prebuilt
cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}-arm64
gate: swift-runner-ios
xcuitest-platform: ios
xcuitest-destination: generic/platform=iOS Simulator
- name: Install the fixture app
id: fixture-app
uses: ./.github/actions/setup-fixture-app
- name: Preflight iOS runner through public CLI
env:
UDID: ${{ steps.ios-simulator.outputs.simulator-udid }}
run: |
# The keyboard must stay expanded for keyboard dismiss to find its Done key.
xcrun simctl spawn "$UDID" defaults write com.apple.keyboard.preferences AutomaticMinimizationEnabled -bool false
pnpm clean:daemon
node bin/agent-device.mjs prepare ios-runner --platform ios --udid "$UDID" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS"
pnpm clean:daemon
- name: Run iOS command perf suite
env:
UDID: ${{ steps.ios-simulator.outputs.simulator-udid }}
APP_ID: ${{ steps.fixture-app.outputs.app-id }}
APP_PATH: ${{ steps.fixture-app.outputs.app-path }}
SCENARIOS: ${{ github.event.inputs.scenarios }}
run: |
pnpm perf --platform ios --udid "$UDID" \
--app-id "$APP_ID" --app-path "$APP_PATH" \
--rounds "$PERF_ROUNDS" --warmup "$PERF_WARMUP" \
${SCENARIOS:+--scenarios "$SCENARIOS"} \
--out-dir "$GITHUB_WORKSPACE/perf-results"
- name: Compare with earlier nights
if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }}
env:
GH_TOKEN: ${{ github.token }}
BASELINE_BRANCH: ${{ github.event.inputs.baseline-branch || 'main' }}
BASELINE_NIGHTS: ${{ github.event.inputs.baseline-nights || '7' }}
run: |
pnpm perf:compare --to perf-results \
--branch "$BASELINE_BRANCH" \
--nights "$BASELINE_NIGHTS" \
--exclude-run "$GITHUB_RUN_ID" \
--out perf-results --github
echo "Full report: the perf-ios-summary artifact holds report.md and summary.json." >> "$GITHUB_STEP_SUMMARY"
- name: Upload iOS perf summary
if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }}
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: perf-ios-summary
path: |
perf-results/summary.json
perf-results/report.md
perf-results/comparison.json
perf-results/comparison.md
retention-days: 90
if-no-files-found: error
- name: Upload iOS perf raw logs
if: ${{ always() }}
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: perf-ios-raw
path: perf-results/raw/
retention-days: 30
if-no-files-found: warn
perf-android:
name: Android Command Perf
if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.android == 'true' }}
runs-on: ubuntu-latest
timeout-minutes: 80
env:
AGENT_DEVICE_PERF_CLI: '--experimental-strip-types src/bin.ts'
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup toolchain
uses: ./.github/actions/setup-node-pnpm
- name: Setup Android replay host
id: android-replay-host
uses: ./.github/actions/setup-android-replay-host
with:
package-helpers: 'true'
- name: Run Android command perf benchmark
uses: reactivecircus/android-emulator-runner@b530d96654c385303d652368551fb075bc2f0b6b # v2.35.0
with:
api-level: 36
arch: x86_64
profile: pixel_7
target: google_apis_playstore
emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim -no-metrics
script: >-
set -e;
adb -s emulator-5554 shell settings put global window_animation_scale 0 || true;
adb -s emulator-5554 shell settings put global transition_animation_scale 0 || true;
adb -s emulator-5554 shell settings put global animator_duration_scale 0 || true;
pnpm perf --platform android --serial emulator-5554 --n "$PERF_ROUNDS" --warmup "$PERF_WARMUP" --out-dir "$GITHUB_WORKSPACE/perf-results"
- name: Upload Android perf report
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: perf-android
path: perf-results/
if-no-files-found: warn