@@ -5,25 +5,29 @@ import { createSnapshotSourceHost } from './host.ts';
55import { readSnapshotSourceToolchain } from './cache-identity.ts' ;
66import { createSnapshotSourceDeadline , type SnapshotSourceDeadline } from './deadline.ts' ;
77import { SnapshotSourceError } from './errors.ts' ;
8- import type { ExecOptions , ExecResult } from '@agent-device/host-kit/command' ;
8+ import { execKillTimeoutError } from './__tests__/exec-timeout-fixture.ts' ;
9+ import {
10+ isCommandTimeoutError ,
11+ type ExecOptions ,
12+ type ExecResult ,
13+ } from '@agent-device/host-kit/command' ;
914import type { SnapshotSourceHost } from './types.ts' ;
1015
11- // Apple's syspolicyd signature scan blocks the first xcodebuild/xcrun exec
12- // after a fresh macOS host boots for roughly 18 to 19 seconds; the immediate
13- // next exec of the same tool is instant (#2422). These cases exercise the
14- // resulting one-retry policy, and the deadline that bounds it, without waiting
15- // on a real cold-start stall: the fake clock only moves when a probe actually
16- // blocks for the timeout it was handed, so a case that claims the budget was
17- // spent had to spend it.
16+ // Apple's syspolicyd signature scan blocks the first exec of an Xcode-owned tool after a fresh
17+ // macOS host boots for roughly 18 to 19 seconds; the immediate next exec of the same tool is
18+ // instant (#2422). These cases exercise the resulting one-retry policy, and the deadline that
19+ // bounds it, without waiting on a real cold-start stall: the fake clock only moves when a probe
20+ // actually blocks for the timeout it was handed, so a case that claims the budget was spent had to
21+ // spend it.
1822
1923test ( 'a cold-start toolchain probe recovers on retry, and the retry gets only what the stall left' , async ( ) => {
2024 const clock = { nowMs : 0 } ;
2125 const timeouts : number [ ] = [ ] ;
2226 let calls = 0 ;
23- const host = fakeToolchainHost ( ( command , args , options ) => {
27+ const host = fakeToolchainHost ( async ( command , args , options ) => {
2428 calls += 1 ;
2529 timeouts . push ( options . timeoutMs ?? 0 ) ;
26- if ( calls === 1 ) throw blockForWholeTimeout ( clock , command , options ) ;
30+ if ( calls === 1 ) throw await blockForWholeTimeout ( clock , options ) ;
2731 return toolchainAnswer ( command , args ) ;
2832 } ) ;
2933
@@ -36,35 +40,77 @@ test('a cold-start toolchain probe recovers on retry, and the retry gets only wh
3640 assert . equal ( identity . xcode , 'Xcode 26.2\nBuild version 17C52' ) ;
3741 assert . equal ( identity . macosBuild , '24G90' ) ;
3842 assert . equal ( identity . architecture , 'arm64' ) ;
39- assert . equal ( identity . simulatorSdk , '26.2' ) ;
43+ assert . equal ( identity . simulatorRuntime , 'iOS 26.2' ) ;
4044 // The stalled first attempt is capped at the 30 s per-probe ceiling; the
4145 // retry runs on the 10 s the shared deadline has left, not a second 30 s.
4246 assert . deepEqual ( timeouts . slice ( 0 , 2 ) , [ 30_000 , 10_000 ] ) ;
4347 assert . equal ( clock . nowMs , 30_000 ) ;
44- // 5 baseline probes (xcodebuild, sw_vers x2, uname, xcrun ) plus the one
48+ // 4 baseline probes (xcodebuild, sw_vers x2, uname) plus the one
4549 // retry that recovered the first, timed-out call.
46- assert . equal ( calls , 6 ) ;
50+ assert . equal ( calls , 5 ) ;
4751} ) ;
4852
49- test ( 'a toolchain host that never returns still fails at the deadline with the same timeout error' , async ( ) => {
53+ // The identity read is allowed to exec one Xcode-owned binary. The Simulator SDK a second
54+ // `xcrun` probe used to report ships inside the selected `Xcode.app`, so it cannot move under a
55+ // `xcodebuild -version` build that already pins it -- and every extra Xcode-owned exec is another
56+ // toolchain the job can wait on and fail against (#2712).
57+ test ( 'the toolchain identity execs one Xcode-owned binary, and no xcrun' , async ( ) => {
58+ const clock = { nowMs : 0 } ;
59+ const probed : string [ ] = [ ] ;
60+ const host = fakeToolchainHost ( ( command , args ) => {
61+ probed . push ( command ) ;
62+ return toolchainAnswer ( command , args ) ;
63+ } ) ;
64+
65+ await readSnapshotSourceToolchain ( host , 'iOS 26.2' , fakeClockDeadline ( 120_000 , clock ) ) ;
66+
67+ assert . deepEqual ( probed , [ 'xcodebuild' , 'sw_vers' , 'sw_vers' , 'uname' ] ) ;
68+ } ) ;
69+
70+ test ( 'a toolchain host that never returns reports the stalled probe after one retry' , async ( ) => {
5071 const clock = { nowMs : 0 } ;
5172 const timeouts : number [ ] = [ ] ;
52- const host = fakeToolchainHost ( ( command , _args , options ) => {
73+ const host = fakeToolchainHost ( async ( _command , _args , options ) => {
5374 timeouts . push ( options . timeoutMs ?? 0 ) ;
54- throw blockForWholeTimeout ( clock , command , options ) ;
75+ throw await blockForWholeTimeout ( clock , options ) ;
5576 } ) ;
5677
5778 await assert . rejects (
5879 readSnapshotSourceToolchain ( host , 'iOS 26.2' , fakeClockDeadline ( 120_000 , clock ) ) ,
59- ( error : unknown ) =>
60- error instanceof AppError && error . message === 'xcodebuild timed out after 30000ms' ,
80+ ( error : unknown ) => {
81+ assertToolchainProbeStall ( error , 'xcodebuild' , [ 30_000 , 30_000 ] ) ;
82+ return true ;
83+ } ,
6184 ) ;
6285 // Exactly one retry, not an unbounded loop, and the retry is charged the
6386 // remainder rather than a fresh ceiling.
6487 assert . deepEqual ( timeouts , [ 30_000 , 30_000 ] ) ;
6588 assert . equal ( clock . nowMs , 60_000 ) ;
6689} ) ;
6790
91+ // The stall names the probe that hit it, not just the first one in the sequence: a host whose
92+ // `sw_vers` answers late must not read as an Xcode problem.
93+ test ( 'a later probe that stalls out names that probe and its own attempts' , async ( ) => {
94+ const clock = { nowMs : 0 } ;
95+ let macosBuildCalls = 0 ;
96+ const host = fakeToolchainHost ( async ( command , args , options ) => {
97+ if ( command !== 'sw_vers' || ! args . includes ( '-buildVersion' ) ) {
98+ return toolchainAnswer ( command , args ) ;
99+ }
100+ macosBuildCalls += 1 ;
101+ throw await blockForWholeTimeout ( clock , options ) ;
102+ } ) ;
103+
104+ await assert . rejects (
105+ readSnapshotSourceToolchain ( host , 'iOS 26.2' , fakeClockDeadline ( 120_000 , clock ) ) ,
106+ ( error : unknown ) => {
107+ assertToolchainProbeStall ( error , 'sw_vers' , [ 30_000 , 30_000 ] ) ;
108+ return true ;
109+ } ,
110+ ) ;
111+ assert . equal ( macosBuildCalls , 2 ) ;
112+ } ) ;
113+
68114test ( 'a probe that failed on its own and merely says "timed out" in its message is not retried' , async ( ) => {
69115 const clock = { nowMs : 0 } ;
70116 let calls = 0 ;
@@ -101,7 +147,7 @@ type ToolchainProbeCancellationCase = {
101147 aborts : 'never' | 'before-the-deadline' | 'while-it-blocks' | 'as-it-unwinds' ;
102148 /** The phase deadline. 30 s is spent in full by one stalled probe, leaving no retry. */
103149 deadlineMs : number ;
104- expected : 'cancelled' | 'exec-timeout ' | 'command-failure' ;
150+ expected : 'cancelled' | 'probe-stall ' | 'command-failure' ;
105151 execs : number ;
106152 clockMs : number ;
107153} ;
@@ -146,12 +192,12 @@ const CANCELLATION_CASES: ToolchainProbeCancellationCase[] = [
146192 clockMs : 30_000 ,
147193 } ,
148194 {
149- // Nothing left to retry on, so the original timeout propagates unchanged .
195+ // Nothing left to retry on, so a single stalled attempt already names the probe .
150196 label : 'never aborted, the first probe spends the whole deadline' ,
151197 firstProbe : 'exec-timeout' ,
152198 aborts : 'never' ,
153199 deadlineMs : 30_000 ,
154- expected : 'exec-timeout ' ,
200+ expected : 'probe-stall ' ,
155201 execs : 1 ,
156202 clockMs : 30_000 ,
157203 } ,
@@ -162,13 +208,13 @@ test.each(CANCELLATION_CASES)('cancellation matrix: $label', async (testCase) =>
162208 const request = new AbortController ( ) ;
163209 if ( testCase . aborts === 'before-the-deadline' ) request . abort ( ) ;
164210 let execs = 0 ;
165- const host = fakeToolchainHost ( ( command , args , options ) => {
211+ const host = fakeToolchainHost ( async ( command , args , options ) => {
166212 execs += 1 ;
167213 if ( execs > 1 || ! testCase . firstProbe ) return toolchainAnswer ( command , args ) ;
168214 if ( testCase . aborts === 'while-it-blocks' ) request . abort ( ) ;
169215 const failure =
170216 testCase . firstProbe === 'exec-timeout'
171- ? blockForWholeTimeout ( clock , command , options )
217+ ? await blockForWholeTimeout ( clock , options )
172218 : // No `timeoutMs` detail: the tool failed on its own, so nothing retries it.
173219 new AppError ( 'COMMAND_FAILED' , `${ command } : unexpected error` , { cmd : command } ) ;
174220 if ( testCase . aborts === 'as-it-unwinds' ) request . abort ( ) ;
@@ -204,13 +250,33 @@ function assertExpectedToolchainFailure(
204250 assert . equal ( error . details ?. reason , 'request_canceled' , testCase . label ) ;
205251 return ;
206252 }
253+ if ( testCase . expected === 'probe-stall' ) {
254+ assertToolchainProbeStall ( error , 'xcodebuild' , [ 30_000 ] ) ;
255+ return ;
256+ }
207257 assert . ok ( error instanceof AppError , `${ testCase . label } : expected the probe's own failure` ) ;
208- assert . equal (
209- error . message ,
210- testCase . expected === 'exec-timeout'
211- ? 'xcodebuild timed out after 30000ms'
212- : 'xcodebuild: unexpected error' ,
213- testCase . label ,
258+ assert . equal ( error . message , 'xcodebuild: unexpected error' , testCase . label ) ;
259+ }
260+
261+ /**
262+ * A probe that stalled out names itself, says what each attempt was armed with, and keeps the exec
263+ * layer's own kill as its cause -- so a job can tell "this tool never answered" from a device
264+ * failure without reading a stack frame (#2712).
265+ */
266+ function assertToolchainProbeStall (
267+ error : unknown ,
268+ command : string ,
269+ attemptTimeoutsMs : number [ ] ,
270+ ) : void {
271+ assert . ok ( error instanceof SnapshotSourceError , `expected a toolchain probe stall, got ${ error } ` ) ;
272+ assert . equal ( error . failureKind , 'timeout' ) ;
273+ assert . equal ( error . failureCode , 'toolchain-probe-stalled' ) ;
274+ assert . equal ( error . details ?. command , command ) ;
275+ assert . deepEqual ( error . details ?. attemptTimeoutsMs , attemptTimeoutsMs ) ;
276+ assert . match ( String ( error . details ?. hint ) , new RegExp ( `run \`${ command } \` by hand` ) ) ;
277+ assert . ok (
278+ isCommandTimeoutError ( error . cause ) ,
279+ 'the exec layer kill the probe hit stays the cause' ,
214280 ) ;
215281}
216282
@@ -219,33 +285,35 @@ function fakeClockDeadline(timeoutMs: number, clock: { nowMs: number }): Snapsho
219285 return createSnapshotSourceDeadline ( timeoutMs , undefined , ( ) => clock . nowMs ) ;
220286}
221287
222- /** A probe that blocked for its whole timeout and was then killed, as the exec layer reports it. */
223- function blockForWholeTimeout (
288+ /**
289+ * A probe that blocked for its whole timeout and was then killed. The fake clock advances by the
290+ * budget the probe was handed, and the failure is the one `exec.ts` really raises for that kill.
291+ */
292+ async function blockForWholeTimeout (
224293 clock : { nowMs : number } ,
225- command : string ,
226294 options : ExecOptions ,
227- ) : AppError {
228- const timeoutMs = options . timeoutMs ?? 0 ;
229- clock . nowMs += timeoutMs ;
230- return new AppError ( 'COMMAND_FAILED' , `${ command } timed out after ${ timeoutMs } ms` , { timeoutMs } ) ;
295+ ) : Promise < unknown > {
296+ clock . nowMs += options . timeoutMs ?? 0 ;
297+ return await execKillTimeoutError ( ) ;
231298}
232299
300+ /**
301+ * What the host answers each probe the identity read is allowed to run. A command outside this list
302+ * is a probe the identity read must not open at all (#2712).
303+ */
233304function toolchainAnswer ( command : string , args : string [ ] ) : ExecResult {
234- const stdout =
235- command === 'xcodebuild'
236- ? 'Xcode 26.2\nBuild version 17C52'
237- : command === 'sw_vers'
238- ? args . includes ( '-buildVersion' )
239- ? '24G90'
240- : '15.6'
241- : command === 'uname'
242- ? 'arm64'
243- : '26.2' ;
244- return { stdout, stderr : '' , exitCode : 0 } ;
305+ if ( command === 'xcodebuild' ) {
306+ return { stdout : 'Xcode 26.2\nBuild version 17C52' , stderr : '' , exitCode : 0 } ;
307+ }
308+ if ( command === 'sw_vers' ) {
309+ return { stdout : args . includes ( '-buildVersion' ) ? '24G90' : '15.6' , stderr : '' , exitCode : 0 } ;
310+ }
311+ if ( command === 'uname' ) return { stdout : 'arm64' , stderr : '' , exitCode : 0 } ;
312+ throw new Error ( `the identity read execed ${ command } ${ args . join ( ' ' ) } ` ) ;
245313}
246314
247315function fakeToolchainHost (
248- run : ( command : string , args : string [ ] , options : ExecOptions ) => ExecResult ,
316+ run : ( command : string , args : string [ ] , options : ExecOptions ) => ExecResult | Promise < ExecResult > ,
249317) : SnapshotSourceHost {
250318 const real = createSnapshotSourceHost ( ) ;
251319 return {
0 commit comments