Skip to content

Commit c0ef0fa

Browse files
committed
refactor(ios-runner): one resend budget per error class, no status probe for structured replies
Review follow-ups on the RUNNER_BUSY resend window: - a structured runner reply already says whether the command ran, so executeRunnerCommand no longer sends a lifecycle status probe for it; the raw refusal reaches the caller's resend policy unwrapped. The predicate (isStructuredRunnerFailure) moves from runner-session.ts to the contract. - the busy refusal is a row in RUNNER_ERROR_RULES (runner_busy_refusal, with a drainResend verdict) so isRunnerBusyError reads the table like every sibling predicate and diagnostics name the refusal instead of a flag. - runner-client.ts states both resend budgets in one place (readOnlyResendBudget) instead of splitting the busy cap into maxAttempts and the transport cap into the predicate; the window is documented as a heuristic with the log marker to tune it by. - the resend tests reuse makeRunnerSession from the shared fixtures and no longer encode status probes between attempts.
1 parent f39d911 commit c0ef0fa

5 files changed

Lines changed: 64 additions & 77 deletions

File tree

‎packages/platform-apple/src/runner/__tests__/runner-command-busy-resend.test.ts‎

Lines changed: 11 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
import { beforeEach, test, vi } from 'vitest';
22
import assert from 'node:assert/strict';
33
import { IOS_SIMULATOR } from './device-fixtures.ts';
4-
import { createTestRequestCancellation } from './runner-session-fixtures.ts';
4+
import { createTestRequestCancellation, makeRunnerSession } from './runner-session-fixtures.ts';
55
import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors';
66
import { appleRunnerTestHost } from '../test-host.ts';
7-
import type { RunnerSession } from '../runner-session-types.ts';
87

98
// The read-only resend policy around a `RUNNER_BUSY` refusal. The runner refuses fast while
109
// watchdog-abandoned XCTest work drains, so the daemon's resend has to outlast that drain, keep
@@ -51,7 +50,7 @@ beforeEach(() => {
5150
isRequestCanceled: requestCancellation.isRequestCanceled,
5251
getRequestSignal: () => undefined,
5352
});
54-
const session = readySession();
53+
const session = makeRunnerSession({ state: 'ready' });
5554
mockEnsureRunnerSession.mockResolvedValue(session);
5655
// A live session, or `executeRunnerCommand` reads the resend as a recycle boot and charges the
5756
// per-request recycle budget instead of resending.
@@ -61,20 +60,6 @@ beforeEach(() => {
6160
});
6261
});
6362

64-
function readySession(): RunnerSession {
65-
return {
66-
sessionId: 'session-8100',
67-
device: IOS_SIMULATOR,
68-
deviceId: IOS_SIMULATOR.id,
69-
port: 8100,
70-
xctestrunPath: '/tmp/runner.xctestrun',
71-
jsonPath: '/tmp/runner.json',
72-
testPromise: Promise.resolve({ exitCode: 0, stdout: '', stderr: '' }),
73-
child: { pid: 1234, exitCode: null },
74-
state: 'ready',
75-
} as RunnerSession;
76-
}
77-
7863
/** The runner's live refusal, exactly as the transport decodes it. */
7964
function busyRefusal(): AppError {
8065
return buildRunnerResponseError({
@@ -83,13 +68,6 @@ function busyRefusal(): AppError {
8368
});
8469
}
8570

86-
/** What the status probe reads back for the same refusal after each resend. */
87-
const busyJournal = {
88-
lifecycleState: 'failed',
89-
lifecycleErrorCode: 'RUNNER_BUSY',
90-
lifecycleErrorMessage: 'The iOS runner is still finishing a previous command',
91-
};
92-
9371
function sentCommands(): string[] {
9472
return mockExecuteRunnerCommandWithSession.mock.calls.map((call) => call[2].command);
9573
}
@@ -100,11 +78,8 @@ test('read-only commands wait out RUNNER_BUSY past the transport resend backoff'
10078
vi.useFakeTimers();
10179
mockExecuteRunnerCommandWithSession
10280
.mockRejectedValueOnce(busyRefusal())
103-
.mockResolvedValueOnce(busyJournal)
10481
.mockRejectedValueOnce(busyRefusal())
105-
.mockResolvedValueOnce(busyJournal)
10682
.mockRejectedValueOnce(busyRefusal())
107-
.mockResolvedValueOnce(busyJournal)
10883
.mockResolvedValueOnce({ nodes: [], truncated: false });
10984

11085
const pending = runAppleRunnerCommand(IOS_SIMULATOR, { command: 'snapshot' });
@@ -113,22 +88,16 @@ test('read-only commands wait out RUNNER_BUSY past the transport resend backoff'
11388

11489
assert.deepEqual(result, { nodes: [], truncated: false });
11590
assert.equal(mockInvalidateRunnerSession.mock.calls.length, 0);
116-
assert.deepEqual(sentCommands(), [
117-
'snapshot',
118-
'status',
119-
'snapshot',
120-
'status',
121-
'snapshot',
122-
'status',
123-
'snapshot',
124-
]);
91+
// A structured refusal already says the command did not run, so no status probe sits between
92+
// the resends.
93+
assert.deepEqual(sentCommands(), ['snapshot', 'snapshot', 'snapshot', 'snapshot']);
12594
});
12695

12796
test('a RUNNER_BUSY resend still ends when the refusals outlast the window', async () => {
12897
vi.useFakeTimers();
129-
mockExecuteRunnerCommandWithSession.mockImplementation(async (_device, _session, command) =>
130-
command.command === 'status' ? busyJournal : Promise.reject(busyRefusal()),
131-
);
98+
mockExecuteRunnerCommandWithSession.mockImplementation(async () => {
99+
throw busyRefusal();
100+
});
132101

133102
const pending = runAppleRunnerCommand(IOS_SIMULATOR, { command: 'snapshot' });
134103
const settled = pending.then(
@@ -140,7 +109,8 @@ test('a RUNNER_BUSY resend still ends when the refusals outlast the window', asy
140109

141110
assert.ok(error instanceof AppError);
142111
assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY');
143-
assert.equal(sentCommands().filter((command) => command === 'snapshot').length, 8);
112+
assert.equal(error.details?.recovery, undefined, 'the raw refusal reaches the caller unwrapped');
113+
assert.deepEqual(sentCommands(), Array<string>(8).fill('snapshot'));
144114
});
145115

146116
test('read-only transport failures keep the three-attempt resend budget', async () => {
@@ -170,7 +140,6 @@ test('a cancelled request wakes the RUNNER_BUSY delay instead of sleeping it out
170140
const controller = new AbortController();
171141
mockExecuteRunnerCommandWithSession
172142
.mockRejectedValueOnce(busyRefusal())
173-
.mockResolvedValueOnce(busyJournal)
174143
.mockResolvedValue({ nodes: [], truncated: false });
175144

176145
const pending = runAppleRunnerCommand(
@@ -189,5 +158,5 @@ test('a cancelled request wakes the RUNNER_BUSY delay instead of sleeping it out
189158
const error = await settled;
190159

191160
assert.ok(isRequestCanceledError(error), `expected a canceled request, got ${String(error)}`);
192-
assert.deepEqual(sentCommands(), ['snapshot', 'status']);
161+
assert.deepEqual(sentCommands(), ['snapshot']);
193162
});

‎packages/platform-apple/src/runner/runner-client.ts‎

Lines changed: 18 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -34,29 +34,23 @@ import { RUNNER_COMMAND_TIMEOUT_MS } from './runner-transport.ts';
3434

3535
// --- Runner command execution ---
3636

37-
/** Transport failures keep the pre-existing three-attempt resend. */
38-
const READ_ONLY_TRANSPORT_RESEND_ATTEMPTS = 3;
39-
4037
/**
41-
* A `RUNNER_BUSY` refusal is the runner refusing fast on purpose while it drains abandoned XCTest
42-
* work (#1105), so the resend has to outlast that drain. An abandoned capture typically drains in
43-
* one to a few seconds, longer than the default 200/400ms transport backoff, which gave up before
44-
* the runner was answering again. Delays run 200, 400, 800 then 1000ms, about 5.4s across eight
45-
* attempts.
38+
* Attempts a read-only command may spend on one error class; 1 means no resend. A `RUNNER_BUSY`
39+
* refusal is the runner refusing fast on purpose while it drains abandoned XCTest work (#1105), so
40+
* that budget has to outlast the drain. The window is a heuristic, not a measurement: with 200ms
41+
* doubling to a 1s cap it spans roughly 4 to 6.5s of delay across eight attempts before jitter and
42+
* round trips, and a drain that outlives it still surfaces as `RUNNER_BUSY` (the runner's own
43+
* `abandonedForSeconds=` marker in runner.log is the evidence for tuning it). Transport failures
44+
* keep the pre-existing three attempts. The budget is positional: the error on each attempt sets
45+
* how many attempts the loop may reach in total, so three busy refusals followed by a transport
46+
* failure resend no further.
4647
*/
47-
const READ_ONLY_RESEND_POLICY = {
48-
maxAttempts: 8,
49-
baseDelayMs: 200,
50-
maxDelayMs: 1_000,
51-
};
48+
const RUNNER_BUSY_RESEND_ATTEMPTS = 8;
49+
const TRANSPORT_RESEND_ATTEMPTS = 3;
5250

53-
/**
54-
* Decides whether a read-only command is sent again after `error` on `attempt`. A busy refusal
55-
* retries across the whole window; anything else keeps the transport resend budget.
56-
*/
57-
function shouldResendReadOnlyRunnerCommand(error: unknown, attempt: number): boolean {
58-
if (isRunnerBusyError(error)) return true;
59-
return attempt < READ_ONLY_TRANSPORT_RESEND_ATTEMPTS && isRetryableRunnerError(error);
51+
function readOnlyResendBudget(error: unknown): number {
52+
if (isRunnerBusyError(error)) return RUNNER_BUSY_RESEND_ATTEMPTS;
53+
return isRetryableRunnerError(error) ? TRANSPORT_RESEND_ATTEMPTS : 1;
6054
}
6155

6256
export async function runAppleRunnerCommand(
@@ -75,10 +69,12 @@ export async function runAppleRunnerCommand(
7569
return provider.runCommand(device, runnerCommand, options);
7670
},
7771
{
78-
...READ_ONLY_RESEND_POLICY,
72+
maxAttempts: RUNNER_BUSY_RESEND_ATTEMPTS,
73+
baseDelayMs: 200,
74+
maxDelayMs: 1_000,
7975
shouldRetry: (error, attempt) => {
8076
assertRunnerRequestActive(options.requestId);
81-
return shouldResendReadOnlyRunnerCommand(error, attempt);
77+
return attempt < readOnlyResendBudget(error);
8278
},
8379
},
8480
// The busy window is seconds long, so a cancelled request must wake the delay instead of

‎packages/platform-apple/src/runner/runner-contract.ts‎

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,8 @@ type RunnerErrorMatch = {
197197
};
198198

199199
const hasRetriableFlag: RunnerErrorDetailsMatch = (details) => details.retriable === true;
200+
const hasRunnerBusyCode: RunnerErrorDetailsMatch = (details) =>
201+
details.runnerErrorCode === RUNNER_BUSY_RUNNER_CODE;
200202
/**
201203
* The host's own `DevToolsSecurity -status` read, published as typed details by the probe that
202204
* takes it. The build-failure rule below keys on this field and never on the probe's message, so
@@ -216,8 +218,14 @@ const hasReadinessPreflightFailure: RunnerErrorDetailsMatch = (details) =>
216218
details.runnerReadinessPreflightFailed === true && !isRequestCanceledDetails(details);
217219

218220
type RunnerErrorVerdicts = {
219-
/** isRetryableRunnerError: transport error worth a same-session resend. */
221+
/**
222+
* isRetryableRunnerError: worth a same-session resend, either a transport failure or a structured
223+
* refusal that executed nothing. Whether a lost response needs status recovery is a separate
224+
* question ({@link isStructuredRunnerFailure}).
225+
*/
220226
retryable?: boolean;
227+
/** isRunnerBusyError: the runner refused fast while abandoned main-thread work drains. */
228+
drainResend?: boolean;
221229
/** shouldRetryRunnerConnectError: connect loop may keep waiting for the runner. */
222230
connectRetry?: boolean;
223231
/** Session-fatal classification: invalidate the cached runner session with this reason. */
@@ -334,6 +342,13 @@ export const RUNNER_ERROR_RULES: readonly RunnerErrorRule[] = [
334342
match: { code: 'DEVICE_NOT_FOUND', details: hasUsbmuxDeviceUnattached },
335343
verdicts: { connectRetry: false },
336344
},
345+
{
346+
// Named before the generic retriable flag so diagnostics say what refused, not that a flag
347+
// was set. Nothing ran: the runner answered before dispatching the command (#1105).
348+
reason: 'runner_busy_refusal',
349+
match: { code: 'COMMAND_FAILED', details: hasRunnerBusyCode },
350+
verdicts: { retryable: true, connectRetry: true, drainResend: true },
351+
},
337352
{
338353
reason: 'flagged_retriable',
339354
match: { code: 'COMMAND_FAILED', details: hasRetriableFlag },
@@ -613,8 +628,7 @@ export function isRetryableRunnerError(err: unknown): boolean {
613628
* spent its wait.
614629
*/
615630
export function isRunnerBusyError(error: unknown): boolean {
616-
if (!(error instanceof AppError)) return false;
617-
return error.details?.runnerErrorCode === RUNNER_BUSY_RUNNER_CODE;
631+
return runnerErrorVerdict(error, 'drainResend') ?? false;
618632
}
619633

620634
/**
@@ -624,14 +638,21 @@ export function isRunnerBusyError(error: unknown): boolean {
624638
* `details.runnerErrorCode`), so family policy reads the typed detail rather than the message.
625639
*/
626640
export function isRunnerMainThreadOccupiedError(error: unknown): boolean {
627-
if (!(error instanceof AppError)) return false;
628-
const runnerErrorCode = error.details?.runnerErrorCode;
641+
if (isRunnerBusyError(error)) return true;
629642
return (
630-
runnerErrorCode === RUNNER_BUSY_RUNNER_CODE ||
631-
runnerErrorCode === MAIN_THREAD_TIMEOUT_RUNNER_CODE
643+
error instanceof AppError && error.details?.runnerErrorCode === MAIN_THREAD_TIMEOUT_RUNNER_CODE
632644
);
633645
}
634646

647+
/**
648+
* True when the runner answered with a structured failure: the reply itself carries the runner's
649+
* payload, so the command's outcome is known and no lifecycle status probe is needed to recover it.
650+
* A transport-shaped failure (aborted body, malformed payload, refused connection) answered nothing.
651+
*/
652+
export function isStructuredRunnerFailure(error: unknown): boolean {
653+
return error instanceof AppError && error.details?.runner !== undefined;
654+
}
655+
635656
/**
636657
* True when usbmuxd answered and the device is simply not attached by cable.
637658
* A CoreDevice-backed device falls back to its network tunnel; an XCTest-backed

‎packages/platform-apple/src/runner/runner-lifecycle.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import {
1616
import {
1717
assertRunnerRequestActive,
1818
isRetryableRunnerError,
19+
isStructuredRunnerFailure,
1920
resolveRunnerRequestSignal,
2021
shouldRebuildCachedRunnerArtifact,
2122
shouldRestartRunnerAfterReadinessPreflight,
@@ -325,7 +326,10 @@ export async function executeRunnerCommand(
325326
recoveredDiagnosticPhase: 'ios_runner_readiness_preflight_recovered',
326327
});
327328
}
328-
if (session && isRetryableRunnerError(appErr)) {
329+
// Status recovery answers "did the command I lost the response to run?". A structured reply
330+
// (a RUNNER_BUSY refusal, for one) already answered, so it is rethrown for the caller's own
331+
// resend policy instead of paying a status round trip per attempt.
332+
if (session && isRetryableRunnerError(appErr) && !isStructuredRunnerFailure(appErr)) {
329333
return await handleRunnerTransportErrorAfterCommandSend({
330334
device,
331335
session,
@@ -392,7 +396,7 @@ async function restartSessionAndRunCommand(params: {
392396
return recovered;
393397
} catch (error) {
394398
const retryAppErr = asAppError(error, 'COMMAND_FAILED');
395-
if (isRetryableRunnerError(retryAppErr)) {
399+
if (isRetryableRunnerError(retryAppErr) && !isStructuredRunnerFailure(retryAppErr)) {
396400
try {
397401
return await handleRunnerTransportErrorAfterCommandSend({
398402
device,

‎packages/platform-apple/src/runner/runner-session.ts‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ import {
4141
type RunnerCommand,
4242
resolveRunnerFatalErrorReason,
4343
isRunnerMainThreadOccupiedError,
44+
isStructuredRunnerFailure,
4445
enrichRunnerStartupFailureWithDeviceStates,
4546
} from './runner-contract.ts';
4647
import {
@@ -975,10 +976,6 @@ function readRunnerMainThreadBusy(data: Record<string, unknown>): boolean | unde
975976
return typeof data.runnerMainThreadBusy === 'boolean' ? data.runnerMainThreadBusy : undefined;
976977
}
977978

978-
function isStructuredRunnerFailure(error: unknown): boolean {
979-
return error instanceof AppError && error.details?.runner !== undefined;
980-
}
981-
982979
function markSkippedPreflightTransportError(
983980
error: unknown,
984981
session: RunnerSession,

0 commit comments

Comments
 (0)