Skip to content

Commit 84f08e9

Browse files
committed
fix(ios-runner): make the busy resend cancellable and split its tests out
Review follow-ups on the RUNNER_BUSY resend window: - pass the request signal to retryWithPolicy so a cancelled request wakes the seconds-long busy delay instead of sleeping it out and sending one more attempt - keep shouldResendReadOnlyRunnerCommand module-private - move the resend tests into runner-command-busy-resend.test.ts; the retry suite is past the size tripwire and may not grow - assert directly that MAIN_THREAD_TIMEOUT and message-only busy errors do not read as a busy refusal
1 parent 0990c75 commit 84f08e9

4 files changed

Lines changed: 223 additions & 93 deletions

File tree

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
import { beforeEach, test, vi } from 'vitest';
2+
import assert from 'node:assert/strict';
3+
import { IOS_SIMULATOR } from './device-fixtures.ts';
4+
import { createTestRequestCancellation } from './runner-session-fixtures.ts';
5+
import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors';
6+
import { appleRunnerTestHost } from '../test-host.ts';
7+
import type { RunnerSession } from '../runner-session-types.ts';
8+
9+
// The read-only resend policy around a `RUNNER_BUSY` refusal (#2804). The runner refuses fast
10+
// while watchdog-abandoned XCTest work drains (1.2s median on the Bluesky bench), so the daemon's
11+
// resend has to outlast that drain, keep the old three-attempt budget for transport failures, and
12+
// stay cancellable across the seconds it may now wait.
13+
14+
const {
15+
mockEnsureRunnerSession,
16+
mockExecuteRunnerCommandWithSession,
17+
mockReadRunnerSessionLiveness,
18+
mockInvalidateRunnerSession,
19+
} = vi.hoisted(() => ({
20+
mockEnsureRunnerSession: vi.fn(),
21+
mockExecuteRunnerCommandWithSession: vi.fn(),
22+
mockReadRunnerSessionLiveness: vi.fn(),
23+
mockInvalidateRunnerSession: vi.fn(),
24+
}));
25+
26+
vi.mock('../runner-session.ts', async () => {
27+
const actual =
28+
await vi.importActual<typeof import('../runner-session.ts')>('../runner-session.ts');
29+
return {
30+
...actual,
31+
ensureRunnerSession: mockEnsureRunnerSession,
32+
executeRunnerCommandWithSession: mockExecuteRunnerCommandWithSession,
33+
readRunnerSessionLiveness: mockReadRunnerSessionLiveness,
34+
invalidateRunnerSession: mockInvalidateRunnerSession,
35+
};
36+
});
37+
38+
import { runAppleRunnerCommand } from '../runner-client.ts';
39+
import { buildRunnerResponseError } from '../runner-contract.ts';
40+
import { resetRunnerRecycleLedgerForTests } from '../runner-recycle-ledger.ts';
41+
42+
const requestCancellation = createTestRequestCancellation();
43+
44+
beforeEach(() => {
45+
vi.resetAllMocks();
46+
vi.useRealTimers();
47+
resetRunnerRecycleLedgerForTests();
48+
requestCancellation.reset();
49+
appleRunnerTestHost.update({
50+
emitDiagnostic: vi.fn(),
51+
isRequestCanceled: requestCancellation.isRequestCanceled,
52+
getRequestSignal: () => undefined,
53+
});
54+
const session = readySession();
55+
mockEnsureRunnerSession.mockResolvedValue(session);
56+
// A live session, or `executeRunnerCommand` reads the resend as a recycle boot and charges the
57+
// per-request recycle budget instead of resending.
58+
mockReadRunnerSessionLiveness.mockReturnValue({
59+
sessionId: session.sessionId,
60+
liveness: 'ready',
61+
});
62+
});
63+
64+
function readySession(): RunnerSession {
65+
return {
66+
sessionId: 'session-8100',
67+
device: IOS_SIMULATOR,
68+
deviceId: IOS_SIMULATOR.id,
69+
port: 8100,
70+
xctestrunPath: '/tmp/runner.xctestrun',
71+
jsonPath: '/tmp/runner.json',
72+
testPromise: Promise.resolve({ exitCode: 0, stdout: '', stderr: '' }),
73+
child: { pid: 1234, exitCode: null },
74+
state: 'ready',
75+
} as RunnerSession;
76+
}
77+
78+
/** The runner's live refusal, exactly as the transport decodes it. */
79+
function busyRefusal(): AppError {
80+
return buildRunnerResponseError({
81+
ok: false,
82+
error: { code: 'RUNNER_BUSY', message: 'The iOS runner is still finishing a previous command' },
83+
});
84+
}
85+
86+
/** What the status probe reads back for the same refusal after each resend. */
87+
const busyJournal = {
88+
lifecycleState: 'failed',
89+
lifecycleErrorCode: 'RUNNER_BUSY',
90+
lifecycleErrorMessage: 'The iOS runner is still finishing a previous command',
91+
};
92+
93+
function sentCommands(): string[] {
94+
return mockExecuteRunnerCommandWithSession.mock.calls.map((call) => call[2].command);
95+
}
96+
97+
test('read-only commands wait out RUNNER_BUSY past the transport resend backoff', async () => {
98+
// The logged shape: three busy answers spaced 200/400/800ms apart, then the runner answers.
99+
// The default three attempts gave up at ~0.6s, 60ms before the abandoned work drained.
100+
vi.useFakeTimers();
101+
mockExecuteRunnerCommandWithSession
102+
.mockRejectedValueOnce(busyRefusal())
103+
.mockResolvedValueOnce(busyJournal)
104+
.mockRejectedValueOnce(busyRefusal())
105+
.mockResolvedValueOnce(busyJournal)
106+
.mockRejectedValueOnce(busyRefusal())
107+
.mockResolvedValueOnce(busyJournal)
108+
.mockResolvedValueOnce({ nodes: [], truncated: false });
109+
110+
const pending = runAppleRunnerCommand(IOS_SIMULATOR, { command: 'snapshot' });
111+
await vi.advanceTimersByTimeAsync(6_000);
112+
const result = await pending;
113+
114+
assert.deepEqual(result, { nodes: [], truncated: false });
115+
assert.equal(mockInvalidateRunnerSession.mock.calls.length, 0);
116+
assert.deepEqual(sentCommands(), [
117+
'snapshot',
118+
'status',
119+
'snapshot',
120+
'status',
121+
'snapshot',
122+
'status',
123+
'snapshot',
124+
]);
125+
});
126+
127+
test('a RUNNER_BUSY resend still ends when the refusals outlast the window', async () => {
128+
vi.useFakeTimers();
129+
mockExecuteRunnerCommandWithSession.mockImplementation(async (_device, _session, command) =>
130+
command.command === 'status' ? busyJournal : Promise.reject(busyRefusal()),
131+
);
132+
133+
const pending = runAppleRunnerCommand(IOS_SIMULATOR, { command: 'snapshot' });
134+
const settled = pending.then(
135+
() => undefined,
136+
(error: unknown) => error,
137+
);
138+
await vi.advanceTimersByTimeAsync(20_000);
139+
const error = await settled;
140+
141+
assert.ok(error instanceof AppError);
142+
assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY');
143+
assert.equal(sentCommands().filter((command) => command === 'snapshot').length, 8);
144+
});
145+
146+
test('read-only transport failures keep the three-attempt resend budget', async () => {
147+
mockExecuteRunnerCommandWithSession
148+
.mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed'))
149+
.mockResolvedValueOnce({ lifecycleState: 'started' })
150+
.mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed'))
151+
.mockResolvedValueOnce({ lifecycleState: 'started' })
152+
.mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed'))
153+
.mockResolvedValueOnce({ lifecycleState: 'started' })
154+
.mockResolvedValueOnce({ nodes: [], truncated: false });
155+
156+
await assert.rejects(
157+
() => runAppleRunnerCommand(IOS_SIMULATOR, { command: 'snapshot' }),
158+
(error: unknown) => {
159+
assert.ok(error instanceof AppError);
160+
assert.equal(error.message, 'fetch failed');
161+
return true;
162+
},
163+
);
164+
165+
assert.equal(sentCommands().filter((command) => command === 'snapshot').length, 3);
166+
});
167+
168+
test('a cancelled request wakes the RUNNER_BUSY delay instead of sleeping it out', async () => {
169+
vi.useFakeTimers();
170+
const controller = new AbortController();
171+
mockExecuteRunnerCommandWithSession
172+
.mockRejectedValueOnce(busyRefusal())
173+
.mockResolvedValueOnce(busyJournal)
174+
.mockResolvedValue({ nodes: [], truncated: false });
175+
176+
const pending = runAppleRunnerCommand(
177+
IOS_SIMULATOR,
178+
{ command: 'snapshot' },
179+
{ signal: controller.signal },
180+
);
181+
const settled = pending.then(
182+
() => undefined,
183+
(error: unknown) => error,
184+
);
185+
// Inside the first 200ms delay; the next attempt would otherwise succeed at the timer.
186+
await vi.advanceTimersByTimeAsync(50);
187+
controller.abort();
188+
await vi.advanceTimersByTimeAsync(0);
189+
const error = await settled;
190+
191+
assert.ok(isRequestCanceledError(error), `expected a canceled request, got ${String(error)}`);
192+
assert.deepEqual(sentCommands(), ['snapshot', 'status']);
193+
});

‎packages/platform-apple/src/runner/__tests__/runner-command-retry.test.ts‎

Lines changed: 0 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import { IOS_SIMULATOR } from './device-fixtures.ts';
44
import { createTestRequestCancellation } from './runner-session-fixtures.ts';
55
import { AppError } from '@agent-device/kernel/errors';
66
import { Deadline } from '../host.ts';
7-
import { buildRunnerResponseError } from '../runner-contract.ts';
87
import { appleRunnerTestHost } from '../test-host.ts';
98
import type { RunnerSession } from '../runner-session-types.ts';
109

@@ -583,97 +582,6 @@ test('read-only commands retry when status shows in-flight work', async () => {
583582
assert.equal(mockExecuteRunnerCommandWithSession.mock.calls[2]?.[2].command, 'snapshot');
584583
});
585584

586-
test('read-only commands wait out RUNNER_BUSY past the transport resend backoff', async () => {
587-
// The Bluesky bench shape: a snapshot right after a recovered capture meets the runner still
588-
// draining the abandoned viewport read. Drain is 1.2s median, so the resend must survive three
589-
// busy answers spaced 200/400/800ms apart; the default three attempts gave up at ~0.6s.
590-
vi.useFakeTimers();
591-
try {
592-
const session = makeRunnerSession({ port: 8100, state: 'ready' });
593-
const busyRefusal = () =>
594-
buildRunnerResponseError({
595-
ok: false,
596-
error: {
597-
code: 'RUNNER_BUSY',
598-
message: 'The iOS runner is still finishing a previous command',
599-
},
600-
});
601-
const busyJournal = {
602-
lifecycleState: 'failed',
603-
lifecycleErrorCode: 'RUNNER_BUSY',
604-
lifecycleErrorMessage: 'The iOS runner is still finishing a previous command',
605-
};
606-
607-
mockEnsureRunnerSession.mockResolvedValue(session);
608-
mockReadRunnerSessionLiveness.mockReturnValue({
609-
sessionId: session.sessionId,
610-
liveness: 'ready',
611-
});
612-
mockExecuteRunnerCommandWithSession
613-
.mockRejectedValueOnce(busyRefusal())
614-
.mockResolvedValueOnce(busyJournal)
615-
.mockRejectedValueOnce(busyRefusal())
616-
.mockResolvedValueOnce(busyJournal)
617-
.mockRejectedValueOnce(busyRefusal())
618-
.mockResolvedValueOnce(busyJournal)
619-
.mockResolvedValueOnce({ nodes: [], truncated: false });
620-
621-
const pending = runAppleRunnerCommand(IOS_SIMULATOR, { command: 'snapshot' });
622-
await vi.advanceTimersByTimeAsync(6_000);
623-
const result = await pending;
624-
625-
assert.deepEqual(result, { nodes: [], truncated: false });
626-
assert.equal(mockInvalidateRunnerSession.mock.calls.length, 0);
627-
const sentCommands = mockExecuteRunnerCommandWithSession.mock.calls.map(
628-
(call) => call[2].command,
629-
);
630-
assert.deepEqual(sentCommands, [
631-
'snapshot',
632-
'status',
633-
'snapshot',
634-
'status',
635-
'snapshot',
636-
'status',
637-
'snapshot',
638-
]);
639-
} finally {
640-
vi.useRealTimers();
641-
}
642-
});
643-
644-
test('read-only transport failures keep the three-attempt resend budget', async () => {
645-
const session = makeRunnerSession({ port: 8100, state: 'ready' });
646-
647-
mockEnsureRunnerSession.mockResolvedValue(session);
648-
mockReadRunnerSessionLiveness.mockReturnValue({
649-
sessionId: session.sessionId,
650-
liveness: 'ready',
651-
});
652-
mockExecuteRunnerCommandWithSession
653-
.mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed'))
654-
.mockResolvedValueOnce({ lifecycleState: 'started' })
655-
.mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed'))
656-
.mockResolvedValueOnce({ lifecycleState: 'started' })
657-
.mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed'))
658-
.mockResolvedValueOnce({ lifecycleState: 'started' })
659-
.mockResolvedValueOnce({ nodes: [], truncated: false });
660-
661-
await assert.rejects(
662-
() => runAppleRunnerCommand(IOS_SIMULATOR, { command: 'snapshot' }),
663-
(error: unknown) => {
664-
assert.ok(error instanceof AppError);
665-
assert.equal(error.message, 'fetch failed');
666-
return true;
667-
},
668-
);
669-
670-
assert.equal(
671-
mockExecuteRunnerCommandWithSession.mock.calls.filter((call) => call[2].command === 'snapshot')
672-
.length,
673-
3,
674-
);
675-
});
676-
677585
test('mutating commands report recovery guidance when completed status has no retained response', async () => {
678586
const session = makeRunnerSession({ port: 8100, state: 'ready' });
679587

‎packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import {
66
RUNNER_ERROR_RULES,
77
RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES,
88
isRetryableRunnerError,
9+
isRunnerBusyError,
910
resolveRunnerFatalErrorReason,
1011
shouldRebuildCachedRunnerArtifact,
1112
shouldRestartRunnerAfterReadinessPreflight,
@@ -228,3 +229,27 @@ test('a refused screen capture keeps the runner reason and stays off the wire co
228229
assert.equal(classified.details.retriable, undefined);
229230
}
230231
});
232+
233+
// --- busy refusal (isRunnerBusyError) ---
234+
235+
test('only the typed RUNNER_BUSY refusal reads as busy', () => {
236+
assert.equal(
237+
isRunnerBusyError(commandFailed('runner is busy', { runnerErrorCode: 'RUNNER_BUSY' })),
238+
true,
239+
);
240+
// The stalling command's own timeout already spent its wait: not a refusal to resend.
241+
assert.equal(
242+
isRunnerBusyError(
243+
commandFailed('main thread execution timed out', { runnerErrorCode: 'MAIN_THREAD_TIMEOUT' }),
244+
),
245+
false,
246+
);
247+
assert.equal(
248+
isRunnerBusyError(
249+
commandFailed('The iOS runner is still finishing a previous command', { retriable: true }),
250+
),
251+
false,
252+
);
253+
assert.equal(isRunnerBusyError(commandFailed('RUNNER_BUSY')), false);
254+
assert.equal(isRunnerBusyError(new Error('RUNNER_BUSY')), false);
255+
});

‎packages/platform-apple/src/runner/runner-client.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import {
1111
assertRunnerRequestActive,
1212
isRetryableRunnerError,
1313
isRunnerBusyError,
14+
resolveRunnerRequestSignal,
1415
withRunnerCommandId,
1516
type RunnerCommand,
1617
} from './runner-contract.ts';
@@ -52,7 +53,7 @@ const READ_ONLY_RESEND_POLICY = {
5253
* Decides whether a read-only command is sent again after `error` on `attempt`. A busy refusal
5354
* retries across the whole window; anything else keeps the transport resend budget.
5455
*/
55-
export function shouldResendReadOnlyRunnerCommand(error: unknown, attempt: number): boolean {
56+
function shouldResendReadOnlyRunnerCommand(error: unknown, attempt: number): boolean {
5657
if (isRunnerBusyError(error)) return true;
5758
return attempt < READ_ONLY_TRANSPORT_RESEND_ATTEMPTS && isRetryableRunnerError(error);
5859
}
@@ -79,6 +80,9 @@ export async function runAppleRunnerCommand(
7980
return shouldResendReadOnlyRunnerCommand(error, attempt);
8081
},
8182
},
83+
// The busy window is seconds long, so a cancelled request must wake the delay instead of
84+
// sleeping it out and sending one more attempt.
85+
{ signal: resolveRunnerRequestSignal(options) },
8286
);
8387
}
8488
return provider.runCommand(device, runnerCommand, options);

0 commit comments

Comments
 (0)