Skip to content

Commit 211f232

Browse files
committed
test(ios): run isolated preparation regression in nightly
1 parent 444cfeb commit 211f232

3 files changed

Lines changed: 93 additions & 17 deletions

File tree

‎.github/workflows/xctest-nightly.yml‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@ name: XCTest Nightly
22

33
# The full iOS runner XCTest suite (#1781 A7). The PR lane (ios.yml) names a hand-written
44
# subset of the target's methods in an `-only-testing:` list; everything outside that list ran
5-
# nowhere at all. This lane drops the filter and runs the test plan whole, skipping only
6-
# `testCommand` — which is the runner's server entry point rather than a test (see the step
7-
# below).
5+
# nowhere at all. This lane runs the test plan whole, except `testCommand` (the runner's
6+
# server entry point). The process-scoped input-preparation regression runs in isolation;
7+
# its result is merged with the remaining suite before checking the executed count.
88
#
99
# No count is quoted here on purpose. `pnpm check:xctest-selection` prints the live split
1010
# (declared / PR-selected / skipped / nightly-only) and is the only place those numbers are
@@ -117,8 +117,21 @@ jobs:
117117
runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
118118
preferred-device-name: iPhone 17 Pro
119119

120-
# Same command as ios.yml's targeted step minus every `-only-testing:` flag, so the
121-
# xctestrun's own test plan decides what runs — with one exception.
120+
# The preparation regression needs a fresh process before any other synthesized
121+
# input. Run it separately, then merge both bundles for the source-derived count.
122+
- name: Run isolated synthesized-input preparation regression
123+
run: |
124+
set -euo pipefail
125+
XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)"
126+
test -n "$XCTESTRUN_PATH"
127+
mkdir -p "$(dirname "$RESULT_BUNDLE_PATH")"
128+
xcodebuild test-without-building \
129+
-xctestrun "$XCTESTRUN_PATH" \
130+
-destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \
131+
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedInputPreparationDoesNotDeliverContactsAndOrdersMixedRoutes \
132+
-resultBundlePath "${RESULT_BUNDLE_PATH%.xcresult}-isolated.xcresult"
133+
134+
# The remaining suite follows the xctestrun plan, excluding the isolated test.
122135
#
123136
# `RunnerTests/testCommand` is not a test. It is the runner's server entry point: it
124137
# opens an NWListener and blocks in `XCTWaiter.wait(timeout: 24 * 60 * 60)` until a
@@ -144,7 +157,15 @@ jobs:
144157
-destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \
145158
-skip-testing:AgentDeviceRunnerUITests/RunnerTests/testCommand \
146159
-skip-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedInputPreparationDoesNotDeliverContactsAndOrdersMixedRoutes \
147-
-resultBundlePath "$RESULT_BUNDLE_PATH"
160+
-resultBundlePath "${RESULT_BUNDLE_PATH%.xcresult}-remaining.xcresult"
161+
162+
- name: Merge isolated and remaining XCTest results
163+
if: always()
164+
run: |
165+
set -euo pipefail
166+
xcrun xcresulttool merge --output-path "$RESULT_BUNDLE_PATH" \
167+
"${RESULT_BUNDLE_PATH%.xcresult}-isolated.xcresult" \
168+
"${RESULT_BUNDLE_PATH%.xcresult}-remaining.xcresult"
148169
149170
# Best-effort and never the job's verdict on its own; the step below is what asserts.
150171
# `--compact` first because a red night's summary is the large one, and the job summary

‎scripts/__tests__/xctest-selection.test.ts‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,13 +84,20 @@ describe('the real tree', () => {
8484
test('the whole-bundle lanes skip the runner server entry point, which is not a test', () => {
8585
// The whole reason -skip-testing: exists in this repo. `testCommand` opens an
8686
// NWListener and waits 24 hours; an unfiltered run would hang the lane to its timeout.
87-
const skipped = loadReport(repoRoot).flagged.filter((entry) => entry.flag === 'skip-testing');
87+
const report = loadReport(repoRoot);
88+
const skipped = report.flagged.filter((entry) => entry.flag === 'skip-testing');
8889
for (const entry of LANES.filter((entry) => entry.selection === 'whole')) {
8990
expect(
9091
skipped.filter((flag) => flag.workflow === entry.workflow).map((flag) => flag.identifier),
9192
).toContain(ENTRY_POINT);
9293
}
93-
for (const entry of skipped) expect(entry.identifier).toBe(ENTRY_POINT);
94+
for (const entry of skipped) {
95+
if (entry.identifier === ENTRY_POINT) continue;
96+
// A real test may be excluded from the shared process only if another invocation
97+
// on the same lane still reaches it. Removing the isolated run must fail this.
98+
const current = LANES.find((lane) => lane.workflow === entry.workflow)!;
99+
expect(report.reach[current.id].has(entry.identifier)).toBe(true);
100+
}
94101
});
95102

96103
test('the simulator-only tests are exactly the ones the macOS build compiles out', () => {
@@ -296,6 +303,40 @@ describe('a planted guard', () => {
296303
});
297304
});
298305

306+
describe('isolated invocations', () => {
307+
const isolated = `${TARGET}/RunnerTests/testIsolated`;
308+
const swift = source(`${ENTRY_SOURCE}extension RunnerTests {\n func testIsolated() {}\n}\n`);
309+
const shared = `xcodebuild test-without-building -skip-testing:${ENTRY_POINT} -skip-testing:${isolated}`;
310+
311+
test('a separate selected invocation restores a test skipped by the shared process', () => {
312+
const report = buildReport(TARGET, swift, [
313+
...laneWorkflows().filter((entry) => entry.workflow !== NIGHTLY_WORKFLOW_FILE),
314+
{
315+
workflow: NIGHTLY_WORKFLOW_FILE,
316+
text: `xcodebuild test-without-building -only-testing:${isolated}\n${shared}`,
317+
},
318+
]);
319+
expect([...report.reach.nightly]).toEqual([isolated]);
320+
expect(report.entryPointReachedBy).toEqual([]);
321+
});
322+
323+
test('removing the isolated invocation leaves that test unreachable on nightly', () => {
324+
const report = buildReport(TARGET, swift, [
325+
...laneWorkflows().filter((entry) => entry.workflow !== NIGHTLY_WORKFLOW_FILE),
326+
{ workflow: NIGHTLY_WORKFLOW_FILE, text: shared },
327+
]);
328+
expect(report.reach.nightly.has(isolated)).toBe(false);
329+
});
330+
331+
test('a skip still wins over an only flag in the same invocation', () => {
332+
const report = buildReport(TARGET, swift, [
333+
...laneWorkflows().filter((entry) => entry.workflow !== NIGHTLY_WORKFLOW_FILE),
334+
{ workflow: NIGHTLY_WORKFLOW_FILE, text: `${shared} -only-testing:${isolated}` },
335+
]);
336+
expect(report.reach.nightly.has(isolated)).toBe(false);
337+
});
338+
});
339+
299340
describe('the workflow scan', () => {
300341
test('reads both flags on the multi-line xcodebuild invocation', () => {
301342
expect(

‎scripts/check-xctest-selection.ts‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -188,19 +188,26 @@ function identifiers(
188188
);
189189
}
190190

191-
/** What one lane reaches: its platform's compiled set, narrowed by its flags. */
191+
/** Union the invocations: a skip in one process must not erase an isolated run. */
192192
function laneReach(
193193
entry: Lane,
194194
declaredTests: readonly DeclaredTest[],
195-
flagged: readonly FlaggedTest[],
195+
workflowText: string,
196196
): Set<string> {
197-
const only = identifiers(flagged, entry.workflow, 'only-testing');
198-
const skipped = identifiers(flagged, entry.workflow, 'skip-testing');
197+
const parts = workflowText.split(/^\s*(?:run:\s*)?xcodebuild\s+test(?:-without-building)?\b/m);
198+
const invocations = parts.length > 1 ? parts.slice(1) : parts;
199+
const compiled = declaredTests
200+
.filter((test) => test.platforms.includes(entry.platform))
201+
.map((test) => test.identifier);
199202
return new Set(
200-
declaredTests
201-
.filter((test) => test.platforms.includes(entry.platform))
202-
.map((test) => test.identifier)
203-
.filter((id) => (entry.selection === 'whole' || only.has(id)) && !skipped.has(id)),
203+
invocations.flatMap((text) => {
204+
const flagged = parseFlaggedTests(entry.workflow, text);
205+
const only = identifiers(flagged, entry.workflow, 'only-testing');
206+
const skipped = identifiers(flagged, entry.workflow, 'skip-testing');
207+
// An explicit list narrows even a whole-bundle lane's isolated invocation.
208+
const whole = entry.selection === 'whole' && only.size === 0;
209+
return compiled.filter((id) => (whole || only.has(id)) && !skipped.has(id));
210+
}),
204211
);
205212
}
206213

@@ -220,7 +227,14 @@ export function buildReport(
220227
// and cannot speak for anything else a workflow might select.
221228
const owned = flagged.filter((entry) => entry.identifier.startsWith(`${target}/`));
222229
const reach = Object.fromEntries(
223-
LANES.map((entry) => [entry.id, laneReach(entry, declaredTests, flagged)]),
230+
LANES.map((entry) => [
231+
entry.id,
232+
laneReach(
233+
entry,
234+
declaredTests,
235+
workflows.find((workflow) => workflow.workflow === entry.workflow)?.text ?? '',
236+
),
237+
]),
224238
) as Record<LaneId, ReadonlySet<string>>;
225239
const entryPoint = `${target}/${ENTRY_POINT_METHOD}`;
226240
const reachedAnywhere = new Set(LANES.flatMap((entry) => [...reach[entry.id]]));

0 commit comments

Comments
 (0)