Skip to content

Commit 420f623

Browse files
committed
fix(ios): budget cold toolchain probes for the first-exec signature stall
xcodebuild/xcrun toolchain probes in cache-identity.ts and runner-cache-metadata.ts were budgeted for a warm toolchain (10s/5s), below the ~18-19s syspolicyd signature-verification stall on the first exec after a fresh macOS host boots. Share one 30s floor constant between both call sites and retry once after a timeout while the deadline allows, since the second exec is instant. Closes #2422
1 parent 6ca66c9 commit 420f623

5 files changed

Lines changed: 234 additions & 11 deletions

File tree

‎packages/platform-apple/src/runner/__tests__/runner-cache-metadata.test.ts‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,60 @@ test('a timed-out probe leaves the toolchain unavailable instead of a comparable
240240
assert.deepEqual(unavailableProbes(), [{ probe: 'xcodebuild -version', reason: 'probe_error' }]);
241241
});
242242

243+
// Apple's syspolicyd signature scan blocks the first xcodebuild/xcrun exec
244+
// after a fresh macOS host boots for roughly 18 to 19 seconds; the immediate
245+
// next exec of the same tool is instant (#2422). These two cases exercise
246+
// the resulting one-retry policy without waiting on a real cold-start stall.
247+
// They use device fixtures untouched by the tests above so the toolchain
248+
// fingerprint cache starts empty for each.
249+
250+
test('a cold-start toolchain probe recovers on retry: the first call exceeds the budget, the second returns immediately', () => {
251+
let xcodebuildAttempts = 0;
252+
runCmdSync.mockImplementation((command: string, args: readonly string[]) => {
253+
if (command === 'xcodebuild') {
254+
xcodebuildAttempts += 1;
255+
if (xcodebuildAttempts === 1) {
256+
throw new AppError('COMMAND_FAILED', 'xcodebuild timed out after 30000ms', {
257+
timeoutMs: 30_000,
258+
});
259+
}
260+
}
261+
return appleToolchainProbeResult(command, args);
262+
});
263+
runCmdSync.mockClear();
264+
265+
const metadata = resolveExpectedRunnerCacheMetadata(IOS_DEVICE);
266+
267+
assert.equal(metadata.xcodeVersion, '26.2');
268+
assert.equal(metadata.xcodeBuildVersion, '17C52');
269+
expect(runCmdSync.mock.calls.filter(([command]) => command === 'xcodebuild')).toHaveLength(2);
270+
});
271+
272+
test('a toolchain host that never returns still fails at the deadline with the same timeout error', () => {
273+
runCmdSync.mockImplementation((command: string, args: readonly string[]) => {
274+
if (command === 'xcodebuild') {
275+
throw new AppError('COMMAND_FAILED', 'xcodebuild timed out after 30000ms', {
276+
timeoutMs: 30_000,
277+
});
278+
}
279+
return appleToolchainProbeResult(command, args);
280+
});
281+
runCmdSync.mockClear();
282+
283+
try {
284+
resolveExpectedRunnerCacheMetadata(MACOS_DEVICE);
285+
assert.fail('expected an always-timing-out toolchain probe to fail the cache decision');
286+
} catch (error) {
287+
assert.ok(error instanceof AppError);
288+
assert.equal(error.code, 'COMMAND_FAILED');
289+
assert.equal(error.details?.retriable, true);
290+
expect(error.message).toContain('xcodebuild -version');
291+
expect(error.message).toContain('xcodebuild timed out after 30000ms');
292+
}
293+
// Exactly one retry, not an unbounded loop: the original attempt plus one retry.
294+
expect(runCmdSync.mock.calls.filter(([command]) => command === 'xcodebuild')).toHaveLength(2);
295+
});
296+
243297
test('a failing probe reports its exit status rather than a fabricated SDK version', () => {
244298
runCmdSync.mockImplementation((command: string, args: readonly string[]) =>
245299
command === 'xcrun'

‎packages/platform-apple/src/runner/runner-cache-metadata.ts‎

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,13 @@ import {
1818
resolveRunnerSdkName,
1919
} from './apple-runner-platform.ts';
2020
import { computeRunnerSourceFingerprint } from './runner-source.ts';
21+
import { COLD_TOOLCHAIN_PROBE_TIMEOUT_MS } from '../toolchain-probe-budget.ts';
2122

2223
const DEFAULT_IOS_RUNNER_APP_BUNDLE_ID = 'com.callstack.agentdevice.runner';
2324
const RUNNER_DERIVED_ROOT = path.join(os.homedir(), '.agent-device', 'apple-runner');
2425
export const RUNNER_CACHE_METADATA_FILE = '.agent-device-runner-cache.json';
2526
const RUNNER_CACHE_SCHEMA_VERSION = 2;
2627
const RUNNER_CACHE_METADATA_VALUE_MAX_LENGTH = 300;
27-
const TOOLCHAIN_PROBE_TIMEOUT_MS = 5_000;
2828
const TOOLCHAIN_PROBE_MAX_BUFFER = 128 * 1024;
2929
const TOOLCHAIN_PROBE_DETAIL_MAX_LENGTH = 200;
3030
const TOOLCHAIN_PROBE_HINT =
@@ -223,11 +223,7 @@ function runToolchainProbe(cmd: string, args: string[]): ProbeResult<string> {
223223
const probe = [cmd, ...args].join(' ');
224224
let output: { exitCode: number; stdout: string; stderr: string };
225225
try {
226-
output = runCmdSync(cmd, args, {
227-
allowFailure: true,
228-
timeoutMs: TOOLCHAIN_PROBE_TIMEOUT_MS,
229-
maxBuffer: TOOLCHAIN_PROBE_MAX_BUFFER,
230-
});
226+
output = runToolchainProbeCommand(cmd, args);
231227
} catch (error) {
232228
return probeFailure(probe, 'probe_error', error instanceof Error ? error.message : `${error}`);
233229
}
@@ -242,6 +238,40 @@ function runToolchainProbe(cmd: string, args: string[]): ProbeResult<string> {
242238
return value ? { ok: true, value } : probeFailure(probe, 'empty_output', 'no output');
243239
}
244240

241+
/**
242+
* Runs one toolchain probe, retrying exactly once if the attempt times out.
243+
* Apple's syspolicyd signature scan blocks the first `xcodebuild`/`xcrun`
244+
* exec after a fresh host boots (see COLD_TOOLCHAIN_PROBE_TIMEOUT_MS); the
245+
* immediate next exec of the same tool is instant, so the retry recovers
246+
* without widening the per-call budget.
247+
*/
248+
function runToolchainProbeCommand(
249+
cmd: string,
250+
args: string[],
251+
): { exitCode: number; stdout: string; stderr: string } {
252+
try {
253+
return execToolchainProbeCommand(cmd, args);
254+
} catch (error) {
255+
if (!isToolchainProbeTimeout(error)) throw error;
256+
return execToolchainProbeCommand(cmd, args);
257+
}
258+
}
259+
260+
function execToolchainProbeCommand(
261+
cmd: string,
262+
args: string[],
263+
): { exitCode: number; stdout: string; stderr: string } {
264+
return runCmdSync(cmd, args, {
265+
allowFailure: true,
266+
timeoutMs: COLD_TOOLCHAIN_PROBE_TIMEOUT_MS,
267+
maxBuffer: TOOLCHAIN_PROBE_MAX_BUFFER,
268+
});
269+
}
270+
271+
function isToolchainProbeTimeout(error: unknown): boolean {
272+
return error instanceof Error && /timed out after \d+ms/.test(error.message);
273+
}
274+
245275
function parseXcodeVersionOutput(
246276
output: ProbeResult<string>,
247277
): ProbeResult<{ version: string; buildVersion: string }> {
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import assert from 'node:assert/strict';
2+
import { test } from 'vitest';
3+
import { AppError } from '@agent-device/kernel/errors';
4+
import { createSnapshotSourceHost } from './host.ts';
5+
import { readSnapshotSourceToolchain } from './cache-identity.ts';
6+
import { createSnapshotSourceDeadline } from './deadline.ts';
7+
import type { SnapshotSourceHost } from './types.ts';
8+
9+
// Apple's syspolicyd signature scan blocks the first xcodebuild/xcrun exec
10+
// after a fresh macOS host boots for roughly 18 to 19 seconds; the immediate
11+
// next exec of the same tool is instant (#2422). These cases exercise the
12+
// resulting one-retry policy without waiting on a real cold-start stall.
13+
14+
test('a cold-start toolchain probe recovers on retry: the first call exceeds the budget, the second returns immediately', async () => {
15+
let calls = 0;
16+
const host = fakeToolchainHost((command, args) => {
17+
calls += 1;
18+
if (calls === 1) {
19+
throw new AppError('COMMAND_FAILED', `${command} timed out after 30000ms`, {
20+
timeoutMs: 30_000,
21+
});
22+
}
23+
return toolchainAnswer(command, args);
24+
});
25+
26+
const identity = await readSnapshotSourceToolchain(
27+
host,
28+
'iOS 26.2',
29+
createSnapshotSourceDeadline(120_000, undefined),
30+
);
31+
32+
assert.equal(identity.xcode, 'Xcode 26.2\nBuild version 17C52');
33+
assert.equal(identity.macosBuild, '24G90');
34+
assert.equal(identity.architecture, 'arm64');
35+
assert.equal(identity.simulatorSdk, '26.2');
36+
// 5 baseline probes (xcodebuild, sw_vers x2, uname, xcrun) plus the one
37+
// retry that recovered the first, timed-out call.
38+
assert.equal(calls, 6);
39+
});
40+
41+
test('a toolchain host that never returns still fails at the deadline with the same timeout error', async () => {
42+
const host = fakeToolchainHost((command) => {
43+
throw new AppError('COMMAND_FAILED', `${command} timed out after 30000ms`, {
44+
timeoutMs: 30_000,
45+
});
46+
});
47+
48+
await assert.rejects(
49+
readSnapshotSourceToolchain(host, 'iOS 26.2', createSnapshotSourceDeadline(120_000, undefined)),
50+
(error: unknown) =>
51+
error instanceof AppError && error.message === 'xcodebuild timed out after 30000ms',
52+
);
53+
});
54+
55+
function toolchainAnswer(
56+
command: string,
57+
args: string[],
58+
): { stdout: string; stderr: string; exitCode: number } {
59+
const stdout =
60+
command === 'xcodebuild'
61+
? 'Xcode 26.2\nBuild version 17C52'
62+
: command === 'sw_vers'
63+
? args.includes('-buildVersion')
64+
? '24G90'
65+
: '15.6'
66+
: command === 'uname'
67+
? 'arm64'
68+
: '26.2';
69+
return { stdout, stderr: '', exitCode: 0 };
70+
}
71+
72+
function fakeToolchainHost(
73+
run: (command: string, args: string[]) => { stdout: string; stderr: string; exitCode: number },
74+
): SnapshotSourceHost {
75+
const real = createSnapshotSourceHost();
76+
return {
77+
...real,
78+
run: async (command, args) => run(command, args),
79+
};
80+
}

‎packages/platform-apple/src/snapshot-source/cache-identity.ts‎

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
import { createHash } from 'node:crypto';
22
import path from 'node:path';
3+
import type { ExecResult } from '@agent-device/host-kit/command';
34
import { snapshotSourceError } from './errors.ts';
45
import { remainingSnapshotSourceMs, type SnapshotSourceDeadline } from './deadline.ts';
56
import type { SnapshotSourceHost } from './types.ts';
7+
import { COLD_TOOLCHAIN_PROBE_TIMEOUT_MS } from '../toolchain-probe-budget.ts';
68

79
export type SnapshotSourceToolchainIdentity = Readonly<{
810
xcode: string;
@@ -84,11 +86,7 @@ async function toolOutput(
8486
args: string[],
8587
deadline: SnapshotSourceDeadline,
8688
): Promise<string> {
87-
const result = await host.run(command, args, {
88-
allowFailure: true,
89-
signal: deadline.signal,
90-
timeoutMs: Math.min(10_000, remainingSnapshotSourceMs(deadline, 'toolchain-probe-deadline')),
91-
});
89+
const result = await runToolchainProbe(host, command, args, deadline);
9290
if (result.exitCode !== 0) {
9391
throw snapshotSourceError('unsupported', 'toolchain-probe-failed', {
9492
command,
@@ -100,3 +98,48 @@ async function toolOutput(
10098
if (!output) throw snapshotSourceError('unsupported', 'toolchain-probe-empty', { command });
10199
return output;
102100
}
101+
102+
/**
103+
* Runs one toolchain probe, retrying exactly once if the attempt times out
104+
* and the deadline still has room. The retry absorbs the cold-start
105+
* signature-verification stall named on COLD_TOOLCHAIN_PROBE_TIMEOUT_MS: the
106+
* first exec of a tool on a fresh host can block for that long, but the
107+
* immediate next exec of the same tool is instant.
108+
*/
109+
async function runToolchainProbe(
110+
host: SnapshotSourceHost,
111+
command: string,
112+
args: string[],
113+
deadline: SnapshotSourceDeadline,
114+
): Promise<ExecResult> {
115+
try {
116+
return await execToolchainProbe(host, command, args, deadline);
117+
} catch (error) {
118+
if (!isToolchainProbeTimeout(error) || !toolchainProbeDeadlineHasRoom(deadline)) throw error;
119+
return await execToolchainProbe(host, command, args, deadline);
120+
}
121+
}
122+
123+
function execToolchainProbe(
124+
host: SnapshotSourceHost,
125+
command: string,
126+
args: string[],
127+
deadline: SnapshotSourceDeadline,
128+
): Promise<ExecResult> {
129+
return host.run(command, args, {
130+
allowFailure: true,
131+
signal: deadline.signal,
132+
timeoutMs: Math.min(
133+
COLD_TOOLCHAIN_PROBE_TIMEOUT_MS,
134+
remainingSnapshotSourceMs(deadline, 'toolchain-probe-deadline'),
135+
),
136+
});
137+
}
138+
139+
function toolchainProbeDeadlineHasRoom(deadline: SnapshotSourceDeadline): boolean {
140+
return !deadline.signal?.aborted && deadline.clock.remainingMs() > 0;
141+
}
142+
143+
function isToolchainProbeTimeout(error: unknown): boolean {
144+
return error instanceof Error && /timed out after \d+ms/.test(error.message);
145+
}
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/**
2+
* Per-call timeout for a toolchain identity probe (`xcodebuild -version`,
3+
* `xcrun --show-sdk-version`, `sw_vers`, `uname`, …). On a fresh macOS host,
4+
* Apple's syspolicyd signature scan blocks the very first `xcodebuild`/
5+
* `xcrun`/large-binary exec after boot for roughly 18 to 19 seconds at 0%
6+
* CPU; the second exec of the same tool is instant. A budget sized for a
7+
* warm toolchain (the old 10 s / 5 s split) trips on that cold-start stall
8+
* and reports a bogus toolchain-probe timeout unrelated to the change under
9+
* test (#2422).
10+
*
11+
* `snapshot-source/cache-identity.ts` and `runner/runner-cache-metadata.ts`
12+
* both read this one constant for their per-call budget, and both retry
13+
* once after a timeout while their deadline still allows it, so the two
14+
* budgets cannot drift apart again.
15+
*/
16+
export const COLD_TOOLCHAIN_PROBE_TIMEOUT_MS = 30_000;

0 commit comments

Comments
 (0)