Repository navigation
242 lines (228 loc) · 13.4 KB
/
Copy pathxctest-nightly.yml
File metadata and controls
242 lines (228 loc) · 13.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
name: XCTest Nightly
# Run all native iOS regressions nightly and on manual dispatch. PRs keep the host
# decision suite, shared platform-branch checks, and public CLI E2E smoke.
# Native fault injection and identity checks remain here; healthy E2E does not
# replace those contracts. The source-derived count guards against a zero-test run.
on:
schedule:
- cron: '30 4 * * *'
workflow_dispatch:
permissions:
contents: read
actions: read
concurrency:
group: ci-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
full-suite:
name: iOS Runner Full XCTest Suite (${{ matrix.leg.name }})
runs-on: macos-26
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
leg:
# Both legs share the toolchain, the runner build cache, and the artifact steps above.
# The full suite runs on an iPhone, where the sole-screen rule is decided by the idiom
# alone. The iPad leg boots an iPad and runs only the sole-screen tests: the iPad half
# of the rule — the route opens only while the in-process full-screen evidence proves
# the app fills its screen — can execute nowhere else, and the evidence line that probe
# prints is what turns a revert to idiom-only routing red instead of silently green.
# `name-prefix` is what each leg asserts the booted simulator's name starts with: the
# boot action falls back to any simulator when the preferred name is missing, so a
# renamed or absent device model must fail the leg that depends on its idiom.
- name: iPhone full suite
device-name: iPhone 17 Pro
name-prefix: iPhone
lane: nightly
- name: iPad sole-screen evidence
device-name: iPad Pro 11-inch (M5)
name-prefix: iPad
lane: nightly-ipad
env:
# Must match apple.yml. The runtime version rides in the derived-data cache key, and
# AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS is BOTH an input to the cache's build-variant
# hash and the `-D AGENT_DEVICE_RUNNER_UNIT_TESTS` compile flag that makes the RunnerTests
# methods exist at all. Dropping it would build a different variant (cache miss) whose
# bundle contains no tests, and `xcodebuild` reports running zero tests as success — the
# "silent 0-test green" this lane would then be. The test-count assertion below is the
# backstop for exactly that. AGENT_DEVICE_XCUITEST_ARCHS rides the cache key too: dropping
# it silently reverts the runner to Xcode's x86_64 default under Rosetta.
IOS_RUNTIME_VERSION: '26.2'
AGENT_DEVICE_XCUITEST_ARCHS: 'arm64'
AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1'
AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived
RESULT_BUNDLE_PATH: ${{ github.workspace }}/.tmp/xctest-nightly/RunnerTests.xcresult
RESULT_SUMMARY_PATH: ${{ github.workspace }}/.tmp/xctest-nightly/test-results-summary.json
XCTEST_LANE: ${{ matrix.leg.lane }}
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup toolchain
uses: ./.github/actions/setup-node-pnpm
with:
cache-dependency-path: |
pnpm-lock.yaml
examples/test-app/pnpm-lock.yaml
# Byte-identical inputs to apple.yml's step, so both lanes restore the same cache entry
# and only the first one to run on a commit pays for the build.
- name: Restore and build iOS XCTest runner
uses: ./.github/actions/setup-apple-runner-build
with:
derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }}
cache-key-prefix: ios-runner-prebuilt
cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}-arm64
gate: swift-runner-ios
xcuitest-platform: ios
xcuitest-destination: generic/platform=iOS Simulator
- name: Boot iOS test simulator
id: ios-simulator
uses: ./.github/actions/boot-ios-test-simulator
with:
runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
preferred-device-name: ${{ matrix.leg.device-name }}
# The boot action falls back to any simulator when the preferred name is missing, so each
# leg proves it booted the family it claims: the full-suite leg's coverage is iPhone-only at
# the idiom, and the iPad leg's evidence line can only come from an iPad. A renamed or
# removed device model on the image fails here instead of quietly testing the wrong idiom.
- name: Assert the booted simulator's family
run: |
set -euo pipefail
node -e '
const { execFileSync } = require("node:child_process");
const [udid, prefix] = process.argv.slice(1);
const payload = JSON.parse(
execFileSync("xcrun", ["simctl", "list", "devices", "-j"], { maxBuffer: 64 * 1024 * 1024 }).toString(),
);
const device = Object.values(payload.devices ?? {})
.flat()
.find((entry) => entry.udid === udid);
if (!device?.name?.startsWith(prefix)) {
console.error(`Booted ${device?.name ?? udid}; this leg needs a simulator named "${prefix}…"`);
process.exit(1);
}
console.log(`Booted ${device.name}`);
' '${{ steps.ios-simulator.outputs.simulator-udid }}' '${{ matrix.leg.name-prefix }}'
shell: bash
# Same command as apple.yml's targeted step minus every `-only-testing:` flag, so the
# xctestrun's own test plan decides what runs — with one exception.
#
# `RunnerTests/testCommand` is not a test. It is the runner's server entry point: it
# opens an NWListener and blocks in `XCTWaiter.wait(timeout: 24 * 60 * 60)` until a
# client closes the session, which is exactly how production drives it
# (packages/platform-apple/src/runner/runner-session.ts always passes it as the sole
# `-only-testing:`). It compiles unconditionally — the `#if AGENT_DEVICE_RUNNER_UNIT_TESTS`
# block ends well above it — so an unfiltered run reaches it in alphabetical order and
# hangs the job until `timeout-minutes`. The method has no in-process escape hatch, so
# `-skip-testing:` is the lever that keeps it out of this run.
#
# A typo in that identifier silently re-arms the hang, so
# `pnpm check:xctest-selection` validates `-skip-testing:` exactly like `-only-testing:`.
- name: Run the full iOS runner XCTest suite
if: matrix.leg.lane == 'nightly'
run: |
set -euo pipefail
XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)"
test -n "$XCTESTRUN_PATH"
mkdir -p "$(dirname "$RESULT_BUNDLE_PATH")"
xcodebuild test-without-building \
-xctestrun "$XCTESTRUN_PATH" \
-destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \
-skip-testing:AgentDeviceRunnerUITests/RunnerTests/testCommand \
-resultBundlePath "$RESULT_BUNDLE_PATH" 2>&1 | tee .tmp/xctest-nightly/runner.log
node --input-type=module -e '
import { readFileSync } from "node:fs";
const log = readFileSync(".tmp/xctest-nightly/runner.log", "utf8");
if (!/\] AGENT_DEVICE_RUNNER_OPTIONAL_PROBE_WAIT_COMPLETED$/m.test(log)) {
throw new Error("Optional observation ended the runner test before its wait completed");
}
'
# The iPad leg's liveness checks are its own assertions below: it runs a named selection, not
# the whole bundle, so the count assertion the full-suite leg makes does not apply to it, and
# `xcodebuild` exits 0 when a selection matches nothing — a renamed test would shrink the run,
# not fail it. The step asserts the executed count equals the selection, then greps the probe
# line the iPad half of the rule prints.
- name: Run the sole-screen tests on iPad
if: matrix.leg.lane == 'nightly-ipad'
run: |
set -euo pipefail
# This leg skips the full-suite step that creates this directory for the iPhone leg, so
# it must create it in its own leg before `tee` writes into it.
mkdir -p .tmp/xctest-nightly
UDID='${{ steps.ios-simulator.outputs.simulator-udid }}'
XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)"
test -n "$XCTESTRUN_PATH"
xcodebuild test-without-building \
-xctestrun "$XCTESTRUN_PATH" \
-destination "platform=iOS Simulator,id=$UDID" \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testLongPressOnAnUnserializableTreeIsSynthesizedOnTheSoleScreen \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testGestureViewportOnAnUnserializableTreeIsTheSoleScreen \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testSoleScreenDisplayIDIsNamedOnlyForAnAppThatFillsItsScreen \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testInAppBackVerifiesSoleScreenWhenXCTestCannotReadFramesOrEnumerate \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testInAppBackAfterARejectedQueryTapsThroughTheRouteThatOpens \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testInAppBackAfterARejectedQueryOnAWindowedIpadTapsThroughTheWindow \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRankedInAppBackUsesSynthesisWithoutXCTestTap \
-resultBundlePath "$RESULT_BUNDLE_PATH" 2>&1 | tee .tmp/xctest-nightly/runner.log
# `xcodebuild` reports an unmatched selection by running fewer tests, not by failing, so
# the executed count from the result bundle is what proves this leg's whole selection ran.
node --input-type=module -e '
import { execFileSync } from "node:child_process";
const summary = JSON.parse(execFileSync(
"xcrun", ["xcresulttool", "get", "test-results", "summary", "--path", process.env.RESULT_BUNDLE_PATH, "--compact"],
{ maxBuffer: 64 * 1024 * 1024 },
).toString());
const expected = 7;
if (summary.totalTestCount !== expected) {
throw new Error(
`The iPad sole-screen selection ran ${summary.totalTestCount} tests, expected ${expected}; ` +
"a selection stopped matching its test",
);
}
'
# The probe prints this line only when a non-phone idiom asks it. Its absence means the
# iPad never probed (a revert to idiom-only routing), not that a test failed.
grep -q 'AGENT_DEVICE_RUNNER_FULL_SCREEN_EVIDENCE result=fills-screen' \
.tmp/xctest-nightly/runner.log \
|| { echo 'The iPad sole-screen route never produced full-screen evidence.'; exit 1; }
# Best-effort and never the job's verdict on its own; the step below is what asserts.
# `--compact` first because a red night's summary is the large one, and the job summary
# has a 1 MiB cap — the plain form is the fallback if an Xcode version rejects the flag.
- name: Extract the test-results summary
if: always()
run: |
set -uo pipefail
[ -d "$RESULT_BUNDLE_PATH" ] || exit 0
xcrun xcresulttool get test-results summary --path "$RESULT_BUNDLE_PATH" --compact \
> "$RESULT_SUMMARY_PATH" 2>/dev/null ||
xcrun xcresulttool get test-results summary --path "$RESULT_BUNDLE_PATH" \
> "$RESULT_SUMMARY_PATH" 2>/dev/null ||
rm -f "$RESULT_SUMMARY_PATH"
exit 0
# Reports the run AND is the lane's own liveness check. `xcodebuild` exits 0 when a
# selection matches nothing, so "green" and "ran no tests" are the same log tail: a
# build variant without the unit-test compile flag, an empty test plan, or a renamed
# target would all read as a healthy night. The reporter asserts the executed count
# equals what `pnpm check:xctest-selection` derives this lane reaches from the `#if`
# guards (XCTEST_LANE above), so a guard that compiles a file of tests out of iOS reads
# as red too. The failure list is capped so a badly red night cannot blow the 1 MiB
# job-summary limit and lose the headline with it.
- name: Report the run and assert the source-derived count
if: always() && matrix.leg.lane == 'nightly'
run: |
set -euo pipefail
if [ ! -s "$RESULT_SUMMARY_PATH" ]; then
echo 'No usable test-results summary was produced, so nothing here can show a test ran.'
echo 'The suite step above failed before or during the run; read its log first.'
echo '### iOS runner full XCTest suite' >> "$GITHUB_STEP_SUMMARY"
echo 'No result-bundle summary; see the run log and the uploaded bundle.' >> "$GITHUB_STEP_SUMMARY"
exit 1
fi
node --experimental-strip-types scripts/xctest-run-summary.ts
- name: Upload result bundle
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: xctest-nightly-${{ matrix.leg.lane }}-${{ github.run_id }}-${{ github.run_attempt }}
include-hidden-files: true
path: .tmp/xctest-nightly
if-no-files-found: warn