Skip to content

Commit 890069c

Browse files
committed
fix(ios-runner): resend read-only commands across the RUNNER_BUSY drain window
1 parent 19cb1fd commit 890069c

6 files changed

Lines changed: 260 additions & 14 deletions

File tree

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
import { beforeEach, test, vi } from 'vitest';
2+
import assert from 'node:assert/strict';
3+
import { IOS_SIMULATOR } from './device-fixtures.ts';
4+
import { createTestRequestCancellation, makeRunnerSession } from './runner-session-fixtures.ts';
5+
import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors';
6+
import { appleRunnerTestHost } from '../test-host.ts';
7+
8+
// The read-only resend policy around a `RUNNER_BUSY` refusal. The runner refuses fast while
9+
// watchdog-abandoned XCTest work drains, so the daemon's resend has to outlast that drain, keep
10+
// the old three-attempt budget for transport failures, and stay cancellable across the seconds it
11+
// may now wait.
12+
13+
const {
14+
mockEnsureRunnerSession,
15+
mockExecuteRunnerCommandWithSession,
16+
mockReadRunnerSessionLiveness,
17+
mockInvalidateRunnerSession,
18+
} = vi.hoisted(() => ({
19+
mockEnsureRunnerSession: vi.fn(),
20+
mockExecuteRunnerCommandWithSession: vi.fn(),
21+
mockReadRunnerSessionLiveness: vi.fn(),
22+
mockInvalidateRunnerSession: vi.fn(),
23+
}));
24+
25+
vi.mock('../runner-session.ts', async () => {
26+
const actual =
27+
await vi.importActual<typeof import('../runner-session.ts')>('../runner-session.ts');
28+
return {
29+
...actual,
30+
ensureRunnerSession: mockEnsureRunnerSession,
31+
executeRunnerCommandWithSession: mockExecuteRunnerCommandWithSession,
32+
readRunnerSessionLiveness: mockReadRunnerSessionLiveness,
33+
invalidateRunnerSession: mockInvalidateRunnerSession,
34+
};
35+
});
36+
37+
import { runAppleRunnerCommand } from '../runner-client.ts';
38+
import { buildRunnerResponseError } from '../runner-contract.ts';
39+
import { resetRunnerRecycleLedgerForTests } from '../runner-recycle-ledger.ts';
40+
41+
const requestCancellation = createTestRequestCancellation();
42+
43+
beforeEach(() => {
44+
vi.resetAllMocks();
45+
vi.useRealTimers();
46+
resetRunnerRecycleLedgerForTests();
47+
requestCancellation.reset();
48+
appleRunnerTestHost.update({
49+
emitDiagnostic: vi.fn(),
50+
isRequestCanceled: requestCancellation.isRequestCanceled,
51+
getRequestSignal: () => undefined,
52+
});
53+
const session = makeRunnerSession({ state: 'ready' });
54+
mockEnsureRunnerSession.mockResolvedValue(session);
55+
// A live session, or `executeRunnerCommand` reads the resend as a recycle boot and charges the
56+
// per-request recycle budget instead of resending.
57+
mockReadRunnerSessionLiveness.mockReturnValue({
58+
sessionId: session.sessionId,
59+
liveness: 'ready',
60+
});
61+
});
62+
63+
/** The runner's live refusal, exactly as the transport decodes it. */
64+
function busyRefusal(): AppError {
65+
return buildRunnerResponseError({
66+
ok: false,
67+
error: { code: 'RUNNER_BUSY', message: 'The iOS runner is still finishing a previous command' },
68+
});
69+
}
70+
71+
function sentCommands(): string[] {
72+
return mockExecuteRunnerCommandWithSession.mock.calls.map((call) => call[2].command);
73+
}
74+
75+
test('read-only commands wait out RUNNER_BUSY past the transport resend backoff', async () => {
76+
// Three busy answers spaced 200/400/800ms apart, then the runner answers. The default three
77+
// attempts stopped after the second delay, before the abandoned work had drained.
78+
vi.useFakeTimers();
79+
mockExecuteRunnerCommandWithSession
80+
.mockRejectedValueOnce(busyRefusal())
81+
.mockRejectedValueOnce(busyRefusal())
82+
.mockRejectedValueOnce(busyRefusal())
83+
.mockResolvedValueOnce({ nodes: [], truncated: false });
84+
85+
const pending = runAppleRunnerCommand(IOS_SIMULATOR, { command: 'snapshot' });
86+
await vi.advanceTimersByTimeAsync(6_000);
87+
const result = await pending;
88+
89+
assert.deepEqual(result, { nodes: [], truncated: false });
90+
assert.equal(mockInvalidateRunnerSession.mock.calls.length, 0);
91+
// A structured refusal already says the command did not run, so no status probe sits between
92+
// the resends.
93+
assert.deepEqual(sentCommands(), ['snapshot', 'snapshot', 'snapshot', 'snapshot']);
94+
});
95+
96+
test('a RUNNER_BUSY resend still ends when the refusals outlast the window', async () => {
97+
vi.useFakeTimers();
98+
mockExecuteRunnerCommandWithSession.mockImplementation(async () => {
99+
throw busyRefusal();
100+
});
101+
102+
const pending = runAppleRunnerCommand(IOS_SIMULATOR, { command: 'snapshot' });
103+
const settled = pending.then(
104+
() => undefined,
105+
(error: unknown) => error,
106+
);
107+
await vi.advanceTimersByTimeAsync(20_000);
108+
const error = await settled;
109+
110+
assert.ok(error instanceof AppError);
111+
assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY');
112+
assert.equal(error.details?.recovery, undefined, 'the raw refusal reaches the caller unwrapped');
113+
assert.deepEqual(sentCommands(), Array<string>(8).fill('snapshot'));
114+
});
115+
116+
test('read-only transport failures keep the three-attempt resend budget', async () => {
117+
mockExecuteRunnerCommandWithSession
118+
.mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed'))
119+
.mockResolvedValueOnce({ lifecycleState: 'started' })
120+
.mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed'))
121+
.mockResolvedValueOnce({ lifecycleState: 'started' })
122+
.mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed'))
123+
.mockResolvedValueOnce({ lifecycleState: 'started' })
124+
.mockResolvedValueOnce({ nodes: [], truncated: false });
125+
126+
await assert.rejects(
127+
() => runAppleRunnerCommand(IOS_SIMULATOR, { command: 'snapshot' }),
128+
(error: unknown) => {
129+
assert.ok(error instanceof AppError);
130+
assert.equal(error.message, 'fetch failed');
131+
return true;
132+
},
133+
);
134+
135+
assert.equal(sentCommands().filter((command) => command === 'snapshot').length, 3);
136+
});
137+
138+
test('a cancelled request wakes the RUNNER_BUSY delay instead of sleeping it out', async () => {
139+
vi.useFakeTimers();
140+
const controller = new AbortController();
141+
mockExecuteRunnerCommandWithSession
142+
.mockRejectedValueOnce(busyRefusal())
143+
.mockResolvedValue({ nodes: [], truncated: false });
144+
145+
const pending = runAppleRunnerCommand(
146+
IOS_SIMULATOR,
147+
{ command: 'snapshot' },
148+
{ signal: controller.signal },
149+
);
150+
const settled = pending.then(
151+
() => undefined,
152+
(error: unknown) => error,
153+
);
154+
// Inside the first 200ms delay; the next attempt would otherwise succeed at the timer.
155+
await vi.advanceTimersByTimeAsync(50);
156+
controller.abort();
157+
await vi.advanceTimersByTimeAsync(0);
158+
const error = await settled;
159+
160+
assert.ok(isRequestCanceledError(error), `expected a canceled request, got ${String(error)}`);
161+
assert.deepEqual(sentCommands(), ['snapshot']);
162+
});

‎packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
import {
99
RUNNER_ERROR_RULES,
1010
isRetryableRunnerError,
11+
isRunnerBusyError,
1112
resolveRunnerFatalErrorReason,
1213
shouldRebuildCachedRunnerArtifact,
1314
shouldRestartRunnerAfterReadinessPreflight,
@@ -230,3 +231,27 @@ test('a refused screen capture keeps the runner reason and stays off the wire co
230231
assert.equal(classified.details.retriable, undefined);
231232
}
232233
});
234+
235+
// --- busy refusal (isRunnerBusyError) ---
236+
237+
test('only the typed RUNNER_BUSY refusal reads as busy', () => {
238+
assert.equal(
239+
isRunnerBusyError(commandFailed('runner is busy', { runnerErrorCode: 'RUNNER_BUSY' })),
240+
true,
241+
);
242+
// The stalling command's own timeout already spent its wait: not a refusal to resend.
243+
assert.equal(
244+
isRunnerBusyError(
245+
commandFailed('main thread execution timed out', { runnerErrorCode: 'MAIN_THREAD_TIMEOUT' }),
246+
),
247+
false,
248+
);
249+
assert.equal(
250+
isRunnerBusyError(
251+
commandFailed('The iOS runner is still finishing a previous command', { retriable: true }),
252+
),
253+
false,
254+
);
255+
assert.equal(isRunnerBusyError(commandFailed('RUNNER_BUSY')), false);
256+
assert.equal(isRunnerBusyError(new Error('RUNNER_BUSY')), false);
257+
});

‎packages/platform-apple/src/runner/runner-client.ts‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,11 @@ import {
99
} from './runner-session.ts';
1010
import {
1111
assertRunnerRequestActive,
12+
resolveRunnerRequestSignal,
1213
withRunnerCommandId,
1314
type RunnerCommand,
1415
} from './runner-contract.ts';
15-
import { isRetryableRunnerError } from './runner-error-classification.ts';
16+
import { isRetryableRunnerError, isRunnerBusyError } from './runner-error-classification.ts';
1617
import { isReadOnlyRunnerCommand } from './runner-command-traits.ts';
1718
import {
1819
createLocalAppleRunnerProvider,
@@ -32,6 +33,25 @@ import { RUNNER_COMMAND_TIMEOUT_MS } from './runner-transport.ts';
3233

3334
// --- Runner command execution ---
3435

36+
/**
37+
* Attempts a read-only command may spend on one error class; 1 means no resend. A `RUNNER_BUSY`
38+
* refusal is the runner refusing fast on purpose while it drains abandoned XCTest work (#1105), so
39+
* that budget has to outlast the drain. The window is a heuristic, not a measurement: with 200ms
40+
* doubling to a 1s cap it spans roughly 4 to 6.5s of delay across eight attempts before jitter and
41+
* round trips, and a drain that outlives it still surfaces as `RUNNER_BUSY` (the runner's own
42+
* `abandonedForSeconds=` marker in runner.log is the evidence for tuning it). Transport failures
43+
* keep the pre-existing three attempts. The budget is positional: the error on each attempt sets
44+
* how many attempts the loop may reach in total, so three busy refusals followed by a transport
45+
* failure resend no further.
46+
*/
47+
const RUNNER_BUSY_RESEND_ATTEMPTS = 8;
48+
const TRANSPORT_RESEND_ATTEMPTS = 3;
49+
50+
function readOnlyResendBudget(error: unknown): number {
51+
if (isRunnerBusyError(error)) return RUNNER_BUSY_RESEND_ATTEMPTS;
52+
return isRetryableRunnerError(error) ? TRANSPORT_RESEND_ATTEMPTS : 1;
53+
}
54+
3555
export async function runAppleRunnerCommand(
3656
device: DeviceInfo,
3757
command: RunnerCommand,
@@ -48,11 +68,17 @@ export async function runAppleRunnerCommand(
4868
return provider.runCommand(device, runnerCommand, options);
4969
},
5070
{
51-
shouldRetry: (error) => {
71+
maxAttempts: RUNNER_BUSY_RESEND_ATTEMPTS,
72+
baseDelayMs: 200,
73+
maxDelayMs: 1_000,
74+
shouldRetry: (error, attempt) => {
5275
assertRunnerRequestActive(options.requestId);
53-
return isRetryableRunnerError(error);
76+
return attempt < readOnlyResendBudget(error);
5477
},
5578
},
79+
// The busy window is seconds long, so a cancelled request must wake the delay instead of
80+
// sleeping it out and sending one more attempt.
81+
{ signal: resolveRunnerRequestSignal(options) },
5682
);
5783
}
5884
return provider.runCommand(device, runnerCommand, options);

‎packages/platform-apple/src/runner/runner-error-classification.ts‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ type RunnerErrorMatch = {
4545
};
4646

4747
const hasRetriableFlag: RunnerErrorDetailsMatch = (details) => details.retriable === true;
48+
const hasRunnerBusyCode: RunnerErrorDetailsMatch = (details) =>
49+
details.runnerErrorCode === RUNNER_BUSY_RUNNER_CODE;
4850
/**
4951
* The host's own `DevToolsSecurity -status` read, published as typed details by the probe that
5052
* takes it. The build-failure rule below keys on this field and never on the probe's message, so
@@ -64,8 +66,14 @@ const hasReadinessPreflightFailure: RunnerErrorDetailsMatch = (details) =>
6466
details.runnerReadinessPreflightFailed === true && !isRequestCanceledDetails(details);
6567

6668
type RunnerErrorVerdicts = {
67-
/** isRetryableRunnerError: transport error worth a same-session resend. */
69+
/**
70+
* isRetryableRunnerError: worth a same-session resend, either a transport failure or a structured
71+
* refusal that executed nothing. Whether a lost response needs status recovery is a separate
72+
* question ({@link isStructuredRunnerFailure}).
73+
*/
6874
retryable?: boolean;
75+
/** isRunnerBusyError: the runner refused fast while abandoned main-thread work drains. */
76+
drainResend?: boolean;
6977
/** shouldRetryRunnerConnectError: connect loop may keep waiting for the runner. */
7078
connectRetry?: boolean;
7179
/** Session-fatal classification: invalidate the cached runner session with this reason. */
@@ -182,6 +190,13 @@ export const RUNNER_ERROR_RULES: readonly RunnerErrorRule[] = [
182190
match: { code: 'DEVICE_NOT_FOUND', details: hasUsbmuxDeviceUnattached },
183191
verdicts: { connectRetry: false },
184192
},
193+
{
194+
// Named before the generic retriable flag so diagnostics say what refused, not that a flag
195+
// was set. Nothing ran: the runner answered before dispatching the command (#1105).
196+
reason: 'runner_busy_refusal',
197+
match: { code: 'COMMAND_FAILED', details: hasRunnerBusyCode },
198+
verdicts: { retryable: true, connectRetry: true, drainResend: true },
199+
},
185200
{
186201
reason: 'flagged_retriable',
187202
match: { code: 'COMMAND_FAILED', details: hasRetriableFlag },
@@ -461,14 +476,31 @@ export function isRetryableRunnerError(err: unknown): boolean {
461476
* `details.runnerErrorCode`), so family policy reads the typed detail rather than the message.
462477
*/
463478
export function isRunnerMainThreadOccupiedError(error: unknown): boolean {
464-
if (!(error instanceof AppError)) return false;
465-
const runnerErrorCode = error.details?.runnerErrorCode;
479+
if (isRunnerBusyError(error)) return true;
466480
return (
467-
runnerErrorCode === RUNNER_BUSY_RUNNER_CODE ||
468-
runnerErrorCode === MAIN_THREAD_TIMEOUT_RUNNER_CODE
481+
error instanceof AppError && error.details?.runnerErrorCode === MAIN_THREAD_TIMEOUT_RUNNER_CODE
469482
);
470483
}
471484

485+
/**
486+
* True when the runner refused this command outright because watchdog-abandoned XCTest work still
487+
* occupies its main thread (`RUNNER_BUSY`). Nothing was executed, so a read-only caller may resend
488+
* once the work drains; `MAIN_THREAD_TIMEOUT` is deliberately excluded because that command already
489+
* spent its wait.
490+
*/
491+
export function isRunnerBusyError(error: unknown): boolean {
492+
return runnerErrorVerdict(error, 'drainResend') ?? false;
493+
}
494+
495+
/**
496+
* True when the runner answered with a structured failure: the reply itself carries the runner's
497+
* payload, so the command's outcome is known and no lifecycle status probe is needed to recover it.
498+
* A transport-shaped failure (aborted body, malformed payload, refused connection) answered nothing.
499+
*/
500+
export function isStructuredRunnerFailure(error: unknown): boolean {
501+
return error instanceof AppError && error.details?.runner !== undefined;
502+
}
503+
472504
/**
473505
* True when usbmuxd answered and the device is simply not attached by cable.
474506
* A CoreDevice-backed device falls back to its network tunnel; an XCTest-backed

‎packages/platform-apple/src/runner/runner-lifecycle.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import {
2121
} from './runner-contract.ts';
2222
import {
2323
isRetryableRunnerError,
24+
isStructuredRunnerFailure,
2425
shouldRebuildCachedRunnerArtifact,
2526
shouldRestartRunnerAfterReadinessPreflight,
2627
shouldRestartRunnerBeforeCommandSend,
@@ -327,7 +328,10 @@ export async function executeRunnerCommand(
327328
recoveredDiagnosticPhase: 'ios_runner_readiness_preflight_recovered',
328329
});
329330
}
330-
if (session && isRetryableRunnerError(appErr)) {
331+
// Status recovery answers "did the command I lost the response to run?". A structured reply
332+
// (a RUNNER_BUSY refusal, for one) already answered, so it is rethrown for the caller's own
333+
// resend policy instead of paying a status round trip per attempt.
334+
if (session && isRetryableRunnerError(appErr) && !isStructuredRunnerFailure(appErr)) {
331335
return await handleRunnerTransportErrorAfterCommandSend({
332336
device,
333337
session,
@@ -394,7 +398,7 @@ async function restartSessionAndRunCommand(params: {
394398
return recovered;
395399
} catch (error) {
396400
const retryAppErr = asAppError(error, 'COMMAND_FAILED');
397-
if (isRetryableRunnerError(retryAppErr)) {
401+
if (isRetryableRunnerError(retryAppErr) && !isStructuredRunnerFailure(retryAppErr)) {
398402
try {
399403
return await handleRunnerTransportErrorAfterCommandSend({
400404
device,

‎packages/platform-apple/src/runner/runner-session.ts‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import {
4343
import {
4444
resolveRunnerFatalErrorReason,
4545
isRunnerMainThreadOccupiedError,
46+
isStructuredRunnerFailure,
4647
enrichRunnerStartupFailureWithDeviceStates,
4748
} from './runner-error-classification.ts';
4849
import {
@@ -977,10 +978,6 @@ function readRunnerMainThreadBusy(data: Record<string, unknown>): boolean | unde
977978
return typeof data.runnerMainThreadBusy === 'boolean' ? data.runnerMainThreadBusy : undefined;
978979
}
979980

980-
function isStructuredRunnerFailure(error: unknown): boolean {
981-
return error instanceof AppError && error.details?.runner !== undefined;
982-
}
983-
984981
function markSkippedPreflightTransportError(
985982
error: unknown,
986983
session: RunnerSession,

0 commit comments

Comments
 (0)