Skip to content

Commit c1c17da

Browse files
committed
fix(ios-runner): dispose a busy runner on close so open boots clean (#2552)
A runner with watchdog-abandoned main-thread work refuses every command until it drains or escalates to RUNNER_WEDGED. Plain close retained it and the next open reused it, so close recovered nothing. The runner now stamps its live main-thread occupancy on every successful response; the daemon mirrors it on the session and, at the close retention decision, disposes a runner reported busy instead of pooling it back. Killing the process is the only way to abort uncancellable XCTest work.
1 parent ad9b906 commit c1c17da

9 files changed

Lines changed: 413 additions & 2 deletions

File tree

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -999,6 +999,14 @@ extension RunnerTests {
999999
case idle
10001000
case busy(abandonedForSeconds: TimeInterval)
10011001
case wedged(abandonedForSeconds: TimeInterval)
1002+
1003+
/// Whether the main thread is occupied by watchdog-abandoned work, for the occupancy stamp that
1004+
/// every successful response carries. Wedged is still occupied: it only differs in that a
1005+
/// restart, not waiting, is the cure.
1006+
var reportsMainThreadBusy: Bool {
1007+
if case .idle = self { return false }
1008+
return true
1009+
}
10021010
}
10031011

10041012
func currentMainThreadBusyState() -> MainThreadBusyState {

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,31 @@ extension RunnerTests {
183183
XCTAssertEqual(stamped.error?.message, "boom")
184184
}
185185

186+
func testStampingCurrentMainThreadBusyPreservesPayload() {
187+
let stamped = Response(ok: true, data: DataPayload(nodes: [], truncated: false))
188+
.stampingCurrentMainThreadBusy(true)
189+
190+
XCTAssertEqual(stamped.ok, true)
191+
XCTAssertEqual(stamped.data?.runnerMainThreadBusy, true)
192+
}
193+
194+
func testStampingCurrentMainThreadBusySkipsErrorResponses() {
195+
let response = Response(ok: false, error: ErrorPayload(code: "RUNNER_BUSY", message: "busy"))
196+
let stamped = response.stampingCurrentMainThreadBusy(true)
197+
198+
XCTAssertEqual(stamped.ok, false)
199+
XCTAssertNil(stamped.data)
200+
XCTAssertEqual(stamped.error?.code, "RUNNER_BUSY")
201+
}
202+
203+
func testMainThreadBusyStateReportsOccupancy() {
204+
XCTAssertFalse(MainThreadBusyState.idle.reportsMainThreadBusy)
205+
XCTAssertTrue(MainThreadBusyState.busy(abandonedForSeconds: 5).reportsMainThreadBusy)
206+
XCTAssertTrue(MainThreadBusyState.wedged(abandonedForSeconds: 200).reportsMainThreadBusy)
207+
XCTAssertEqual(
208+
Response(ok: true).stampingCurrentMainThreadBusy(false).data?.runnerMainThreadBusy, false)
209+
}
210+
186211
func testJournalStoredResponseStaysUnstamped() throws {
187212
let journal = RunnerCommandJournal()
188213
let recordStart = runnerJournalCommand("recordStart", id: "record-start-anchor")

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,16 @@ extension Response {
229229
payload.currentUptimeMs = value
230230
return Response(ok: ok, data: payload, error: error)
231231
}
232+
233+
// The daemon reads this occupancy flag to decide whether a healthy response proves the runner
234+
// drained its watchdog-abandoned main-thread work. Only successful responses carry it; a refusal
235+
// is itself the busy signal and needs no stamp.
236+
func stampingCurrentMainThreadBusy(_ value: Bool) -> Response {
237+
guard ok else { return self }
238+
var payload = data ?? DataPayload()
239+
payload.runnerMainThreadBusy = value
240+
return Response(ok: ok, data: payload, error: error)
241+
}
232242
}
233243

234244
struct DataPayload: Codable {
@@ -276,6 +286,11 @@ struct DataPayload: Codable {
276286
var textEntryRoute: String?
277287
var runnerFatal: Bool?
278288
var runnerFatalReason: String?
289+
/// Whether main-thread XCTest work past the execution watchdog is still draining when this
290+
/// response is written. A private-AX snapshot can be served successfully while an abandoned tree
291+
/// crawl still grinds, so the healthy response must carry the live occupancy rather than let the
292+
/// daemon read `ok` as proof the runner drained (#2552).
293+
var runnerMainThreadBusy: Bool?
279294
var completedSteps: Int?
280295
var failedStepIndex: Int?
281296
var sequenceResults: [SequenceStepResult]?

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,9 @@ extension RunnerTests {
274274
// rather than pairing a stale uptime with a much-later receipt time.
275275
let stamped =
276276
response.ok
277-
? response.stampingCurrentUptimeMs(ProcessInfo.processInfo.systemUptime * 1000)
277+
? response
278+
.stampingCurrentUptimeMs(ProcessInfo.processInfo.systemUptime * 1000)
279+
.stampingCurrentMainThreadBusy(currentMainThreadBusyState().reportsMainThreadBusy)
278280
: response
279281
let encoder = JSONEncoder()
280282
let body = (try? encoder.encode(stamped)).flatMap { String(data: $0, encoding: .utf8) } ?? "{}"

‎docs/adr/0005-ios-runner-interaction-lifecycle.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,17 @@ whether the runner is alive and accepting new HTTP requests.
5656
Dead cached runner processes are invalidated without graceful `shutdown`. A process that already
5757
stopped cannot answer the shutdown request, so graceful cleanup only adds stale-listener delay.
5858

59+
The runner stamps its live main-thread occupancy (`runnerMainThreadBusy`) onto every successful
60+
response, and the daemon mirrors it on the `RunnerSession`. A `RUNNER_BUSY` refusal also sets it. This
61+
occupancy is a per-response fact, not a health inference: a private-AX snapshot served successfully
62+
while a watchdog-abandoned tree crawl still grinds on the XCTest main thread reports busy, so a healthy
63+
`ok` is never read as proof the runner drained (#2552). A close that would retain a runner for reuse
64+
instead disposes it when that occupancy is set, because a runner still finishing watchdog-abandoned
65+
work refuses every command until it drains or escalates to `RUNNER_WEDGED`; pooling it back to the next
66+
`open` hands the same stalled runner to the caller and `close` recovers nothing. Killing the process is
67+
the only way to abort uncancellable XCTest work. This is the `RUNNER_WEDGED` restart from #1105 applied
68+
at the close boundary rather than after the wedge threshold elapses.
69+
5970
When XCTest reports a root accessibility snapshot failure such as `kAXErrorIllegalArgument`, the
6071
runner treats the cached app target as suspect. Interactive snapshots fail closed to a truncated
6172
root-only payload instead of issuing more flat fallback queries against the same broken tree, and
@@ -84,6 +95,10 @@ If xcodebuild still exits for another reason, the next command detects the stale
8495
process/liveness checks and avoids the old 15-second graceful-shutdown wait. The remaining latency is
8596
fresh xcodebuild runner startup, not a stale transport stall.
8697

98+
A `close` then `open` on a runner still draining abandoned main-thread work now pays a fresh runner boot
99+
instead of inheriting the stalled one, so the wedge no longer survives the close/open cycle. Runners
100+
that were never reported busy keep the existing warm-reuse path unchanged.
101+
87102
The daemon no longer models a generic "recent success" cache as a runner-health signal. A proven
88103
healthy mutating response for the same app — recorded only after the `runnerFatal` check and only
89104
for allowlisted interactions — is now a real end-to-end liveness proof (HTTP listener through to the
Lines changed: 283 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,283 @@
1+
import assert from 'node:assert/strict';
2+
import { beforeEach, test, vi } from 'vitest';
3+
import { IOS_SIMULATOR } from './device-fixtures.ts';
4+
import { appleRunnerTestHost } from '../test-host.ts';
5+
import {
6+
makeClassifyOwnerLivenessViaMocks,
7+
makeBackgroundRunner,
8+
makeRunnerSession,
9+
runnerError,
10+
runnerResponse,
11+
} from './runner-session-fixtures.ts';
12+
import { mkdtempForTestSync } from './tmp-dir.ts';
13+
14+
// The main-thread-occupancy half of the runner-session lifecycle: what the daemon records from a
15+
// runner's busy signals, and what `close` does with a runner that reported abandoned main-thread
16+
// work still draining (#2552). Split out of `runner-session.test.ts`, which already mirrors the
17+
// rest of `runner-session.ts` and is held at the test-file size tripwire.
18+
19+
const {
20+
mockAcquireXcodebuildSimulatorSetRedirect,
21+
mockCleanupTempFile,
22+
mockEnsureXctestrunArtifact,
23+
mockGetFreePort,
24+
mockIsProcessAlive,
25+
mockIsProcessGroupAlive,
26+
mockPrepareXctestrunWithEnv,
27+
mockReadProcessCommand,
28+
mockReadProcessStartTime,
29+
mockResolveExpectedRunnerCacheMetadata,
30+
mockResolveRunnerDerivedPath,
31+
mockRunAppleToolCommand,
32+
mockRunCmdBackground,
33+
mockRunXcrun,
34+
mockSendRunnerCommandOnce,
35+
mockSignalPidsBestEffort,
36+
mockSignalProcessGroupBestEffort,
37+
mockWaitForRunner,
38+
mockRedirectRelease,
39+
} = vi.hoisted(() => ({
40+
mockAcquireXcodebuildSimulatorSetRedirect: vi.fn(),
41+
mockCleanupTempFile: vi.fn(),
42+
mockEnsureXctestrunArtifact: vi.fn(),
43+
mockGetFreePort: vi.fn(),
44+
mockIsProcessAlive: vi.fn(),
45+
mockIsProcessGroupAlive: vi.fn(),
46+
mockPrepareXctestrunWithEnv: vi.fn(),
47+
mockReadProcessCommand: vi.fn((_pid: number) => null as string | null),
48+
mockReadProcessStartTime: vi.fn((_pid: number) => 'fixed-test-owner-start-time' as string | null),
49+
mockResolveExpectedRunnerCacheMetadata: vi.fn(),
50+
mockResolveRunnerDerivedPath: vi.fn(),
51+
mockRunAppleToolCommand: vi.fn(),
52+
mockRunCmdBackground: vi.fn(),
53+
mockRunXcrun: vi.fn(),
54+
mockSendRunnerCommandOnce: vi.fn(),
55+
mockSignalPidsBestEffort: vi.fn(),
56+
mockSignalProcessGroupBestEffort: vi.fn(),
57+
mockWaitForRunner: vi.fn(),
58+
mockRedirectRelease: vi.fn(),
59+
}));
60+
61+
const TEST_OWNER_START_TIME = 'fixed-test-owner-start-time';
62+
63+
vi.mock('../runner-io.ts', async () => {
64+
const actual = await vi.importActual<typeof import('../runner-io.ts')>('../runner-io.ts');
65+
return {
66+
...actual,
67+
cleanupTempFile: mockCleanupTempFile,
68+
getFreePort: mockGetFreePort,
69+
};
70+
});
71+
72+
vi.mock('../runner-startup-transport.ts', async () => {
73+
const actual = await vi.importActual<typeof import('../runner-startup-transport.ts')>(
74+
'../runner-startup-transport.ts',
75+
);
76+
return {
77+
...actual,
78+
waitForRunner: mockWaitForRunner,
79+
};
80+
});
81+
82+
vi.mock('../runner-transport.ts', async () => {
83+
const actual =
84+
await vi.importActual<typeof import('../runner-transport.ts')>('../runner-transport.ts');
85+
return {
86+
...actual,
87+
sendRunnerCommandOnce: mockSendRunnerCommandOnce,
88+
};
89+
});
90+
91+
vi.mock('../runner-xctestrun.ts', async () => {
92+
const actual =
93+
await vi.importActual<typeof import('../runner-xctestrun.ts')>('../runner-xctestrun.ts');
94+
return {
95+
...actual,
96+
acquireXcodebuildSimulatorSetRedirect: mockAcquireXcodebuildSimulatorSetRedirect,
97+
ensureXctestrunArtifact: mockEnsureXctestrunArtifact,
98+
prepareXctestrunWithEnv: mockPrepareXctestrunWithEnv,
99+
resolveExpectedRunnerCacheMetadata: mockResolveExpectedRunnerCacheMetadata,
100+
resolveRunnerDerivedPath: mockResolveRunnerDerivedPath,
101+
};
102+
});
103+
104+
import {
105+
abortAllIosRunnerSessions,
106+
cancelIosRunnerIdleStop,
107+
ensureRunnerSession,
108+
scheduleIosRunnerIdleStop,
109+
executeRunnerCommandWithSession,
110+
getRunnerSessionSnapshot,
111+
} from '../runner-session.ts';
112+
113+
// Test-only stand-in for the daemon's runtime lease-owner-state-dir setter (root-only; the package
114+
// cannot import it). Backs the host.leaseOwnerStateDir() getter the package reads instead.
115+
let leaseOwnerStateDirOverride: string | undefined;
116+
117+
beforeEach(async () => {
118+
appleRunnerTestHost.update({
119+
runCmdBackground: mockRunCmdBackground,
120+
isProcessAlive: mockIsProcessAlive,
121+
isProcessGroupAlive: mockIsProcessGroupAlive,
122+
readProcessCommand: mockReadProcessCommand,
123+
readProcessStartTime: mockReadProcessStartTime,
124+
signalPidsBestEffort: mockSignalPidsBestEffort,
125+
signalProcessGroupBestEffort: mockSignalProcessGroupBestEffort,
126+
runAppleToolCommand: mockRunAppleToolCommand,
127+
runXcrun: mockRunXcrun,
128+
leaseOwnerStateDir: () => leaseOwnerStateDirOverride,
129+
classifyOwnerLiveness: makeClassifyOwnerLivenessViaMocks({
130+
isProcessAlive: (pid) => Boolean(mockIsProcessAlive(pid)),
131+
readProcessStartTime: (pid) => (mockReadProcessStartTime(pid) as string | null) ?? null,
132+
}),
133+
});
134+
await abortAllIosRunnerSessions();
135+
vi.resetAllMocks();
136+
leaseOwnerStateDirOverride = undefined;
137+
process.env.AGENT_DEVICE_IOS_RUNNER_LEASE_DIR = mkdtempForTestSync(
138+
'agent-device-runner-close-test-',
139+
);
140+
mockRunXcrun.mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' });
141+
mockEnsureXctestrunArtifact.mockResolvedValue({
142+
xctestrunPath: '/tmp/base-runner.xctestrun',
143+
derived: '/tmp/derived',
144+
cache: 'miss',
145+
artifact: 'rebuilt',
146+
buildMs: 12,
147+
xctestrunPathSource: 'build',
148+
});
149+
mockGetFreePort.mockResolvedValue(8123);
150+
mockPrepareXctestrunWithEnv.mockResolvedValue({
151+
xctestrunPath: '/tmp/session-runner.xctestrun',
152+
jsonPath: '/tmp/session-runner.json',
153+
});
154+
mockResolveExpectedRunnerCacheMetadata.mockReturnValue({ schemaVersion: 1 });
155+
mockResolveRunnerDerivedPath.mockReturnValue('/tmp/derived');
156+
mockAcquireXcodebuildSimulatorSetRedirect.mockResolvedValue({ release: mockRedirectRelease });
157+
mockRunCmdBackground.mockReturnValue(makeBackgroundRunner(4242));
158+
mockRunAppleToolCommand.mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' });
159+
mockIsProcessAlive.mockReturnValue(true);
160+
mockIsProcessGroupAlive.mockReturnValue(false);
161+
mockReadProcessCommand.mockReturnValue(null);
162+
mockReadProcessStartTime.mockImplementation((pid: number) =>
163+
pid === process.pid ? TEST_OWNER_START_TIME : null,
164+
);
165+
mockWaitForRunner.mockResolvedValue(runnerResponse({ uptimeMs: 1 }));
166+
});
167+
168+
async function waitForRunnerSessionGone(deviceId: string): Promise<boolean> {
169+
const deadline = Date.now() + 1000;
170+
while (Date.now() < deadline) {
171+
if (getRunnerSessionSnapshot(deviceId) === null) return true;
172+
await new Promise((resolve) => setTimeout(resolve, 10));
173+
}
174+
return getRunnerSessionSnapshot(deviceId) === null;
175+
}
176+
177+
test('a RUNNER_BUSY refusal records the runner as still draining main-thread work', async () => {
178+
const session = makeRunnerSession({ ready: true });
179+
mockWaitForRunner.mockResolvedValueOnce(runnerResponse({ uptimeMs: 42 }));
180+
mockSendRunnerCommandOnce.mockResolvedValueOnce(
181+
runnerError({
182+
code: 'RUNNER_BUSY',
183+
message: 'The iOS runner is still finishing a previous command that exceeded its watchdog',
184+
}),
185+
);
186+
187+
await assert.rejects(() =>
188+
executeRunnerCommandWithSession(
189+
IOS_SIMULATOR,
190+
session,
191+
{ command: 'tap', x: 120, y: 240, appBundleId: 'com.example.demo' },
192+
'/tmp/runner.log',
193+
30_000,
194+
),
195+
);
196+
197+
assert.equal(session.runnerMainThreadBusy, true);
198+
});
199+
200+
test('a healthy response stamped with busy main-thread work keeps the session marked busy', async () => {
201+
const session = makeRunnerSession({ ready: true });
202+
mockWaitForRunner.mockResolvedValueOnce(runnerResponse({ uptimeMs: 42 }));
203+
mockSendRunnerCommandOnce.mockResolvedValueOnce(
204+
runnerResponse({ tapped: true, runnerMainThreadBusy: true }),
205+
);
206+
207+
await executeRunnerCommandWithSession(
208+
IOS_SIMULATOR,
209+
session,
210+
{ command: 'tap', x: 120, y: 240, appBundleId: 'com.example.demo' },
211+
'/tmp/runner.log',
212+
30_000,
213+
);
214+
assert.equal(session.runnerMainThreadBusy, true);
215+
216+
mockWaitForRunner.mockResolvedValueOnce(runnerResponse({ uptimeMs: 43 }));
217+
mockSendRunnerCommandOnce.mockResolvedValueOnce(
218+
runnerResponse({ tapped: true, runnerMainThreadBusy: false }),
219+
);
220+
await executeRunnerCommandWithSession(
221+
IOS_SIMULATOR,
222+
session,
223+
{ command: 'tap', x: 130, y: 250, appBundleId: 'com.example.demo' },
224+
'/tmp/runner.log',
225+
30_000,
226+
);
227+
assert.equal(session.runnerMainThreadBusy, false);
228+
});
229+
230+
test('an unstamped healthy response leaves a busy main-thread report intact', async () => {
231+
// Recovered and journal-replayed responses are written without the occupancy stamp; their
232+
// absence must not be read as "the runner drained" (#2552).
233+
const session = makeRunnerSession({ ready: true, runnerMainThreadBusy: true });
234+
mockWaitForRunner.mockResolvedValueOnce(runnerResponse({ uptimeMs: 42 }));
235+
mockSendRunnerCommandOnce.mockResolvedValueOnce(runnerResponse({ tapped: true }));
236+
237+
await executeRunnerCommandWithSession(
238+
IOS_SIMULATOR,
239+
session,
240+
{ command: 'tap', x: 120, y: 240, appBundleId: 'com.example.demo' },
241+
'/tmp/runner.log',
242+
30_000,
243+
);
244+
245+
assert.equal(session.runnerMainThreadBusy, true);
246+
});
247+
248+
test('close disposes a runner still draining main-thread work instead of pooling it for reopen (#2552)', async () => {
249+
const device = { ...IOS_SIMULATOR, id: 'runner-session-close-busy-sim' };
250+
const previousIdleMs = process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS;
251+
// A long idle window proves the teardown below is the busy-dispose branch, not the idle timer.
252+
process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = '300000';
253+
try {
254+
const session = await ensureRunnerSession(device, {});
255+
session.runnerMainThreadBusy = true;
256+
257+
scheduleIosRunnerIdleStop(device.id);
258+
259+
assert.ok(await waitForRunnerSessionGone(device.id));
260+
} finally {
261+
cancelIosRunnerIdleStop(device.id);
262+
if (previousIdleMs === undefined) delete process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS;
263+
else process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = previousIdleMs;
264+
}
265+
});
266+
267+
test('close retains an idle runner that never reported busy main-thread work', async () => {
268+
const device = { ...IOS_SIMULATOR, id: 'runner-session-close-idle-sim' };
269+
const previousIdleMs = process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS;
270+
process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = '300000';
271+
try {
272+
await ensureRunnerSession(device, {});
273+
274+
scheduleIosRunnerIdleStop(device.id);
275+
276+
await new Promise((resolve) => setTimeout(resolve, 100));
277+
assert.ok(getRunnerSessionSnapshot(device.id));
278+
} finally {
279+
cancelIosRunnerIdleStop(device.id);
280+
if (previousIdleMs === undefined) delete process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS;
281+
else process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = previousIdleMs;
282+
}
283+
});

0 commit comments

Comments
 (0)