Skip to content

Commit 96727a0

Browse files
authored
fix(apple-runner): never compare an unavailable toolchain probe; name the mismatching cache keys (#2306)
* fix(apple-runner): never compare an unavailable toolchain probe A timed-out or failed `xcodebuild -version` / `xcrun --show-sdk-*` probe used to fall back to the literal `unknown`, which was memoized for the process and then persisted into the rebuilt cache's metadata, so every later daemon on a healthy host mismatched again and paid a full build-for-testing. Unavailability is now a distinct outcome with no comparable value: only successful probes are memoized, an unreadable toolchain fails the cache decision with a retriable typed error naming the probe that could not answer, and the CI metadata writer refuses to persist a probe it could not read. The cache_metadata_mismatch diagnostic now lists the differing keys with expected and actual values instead of only saying the metadata differed. The runner-source fingerprint moves to the module that owns the runner's source roots, keeping the cache-metadata module within its size budget without adding a module to the Apple facades' eager closure. * refactor(apple-runner): home the fingerprint tests and the rebuild-decision glue Tests mirror source topology: the runner-source fingerprint tests move with the function into runner-source.test.ts and call it directly instead of reaching it through resolveExpectedRunnerCacheMetadata. The rebuild diagnostic's mismatch details move next to the cache state that carries them, so runner-artifact.ts — already past the 500-line extract threshold — gains no behavior. * fix(apple-runner): memoize only a parsed toolchain fingerprint runToolchainProbe cached every nonempty zero-exit answer before parseXcodeVersionOutput could classify it, so a transient malformed xcodebuild answer stayed cached and every later cache decision in the process kept failing after the host recovered. The memo now holds the complete parsed fingerprint per SDK, written only after all three probes answered and parsed; a failed round keeps nothing, so the next request re-probes. Tests cover malformed-to-healthy recovery in one process without resetting the memo, and that a partial round is not kept. * style(apple-runner): oxfmt the cache-metadata module and its tests
1 parent 51bed41 commit 96727a0

9 files changed

Lines changed: 789 additions & 292 deletions
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import { beforeEach, vi } from 'vitest';
2+
import { appleRunnerTestHost } from '../test-host.ts';
3+
import type { ExecResult } from '../host.ts';
4+
5+
const STUBBED_APPLE_TOOLCHAIN = {
6+
xcodeVersion: '26.2',
7+
xcodeBuildVersion: '17C52',
8+
sdkVersion: '26.2',
9+
sdkBuildVersion: '23C53',
10+
} as const;
11+
12+
export function appleToolchainProbeResult(command: string, args: readonly string[]): ExecResult {
13+
if (command === 'xcodebuild' && args[0] === '-version') {
14+
return {
15+
exitCode: 0,
16+
stdout: `Xcode ${STUBBED_APPLE_TOOLCHAIN.xcodeVersion}\nBuild version ${STUBBED_APPLE_TOOLCHAIN.xcodeBuildVersion}\n`,
17+
stderr: '',
18+
};
19+
}
20+
if (command === 'xcrun' && args.includes('--show-sdk-build-version')) {
21+
return { exitCode: 0, stdout: `${STUBBED_APPLE_TOOLCHAIN.sdkBuildVersion}\n`, stderr: '' };
22+
}
23+
if (command === 'xcrun' && args.includes('--show-sdk-version')) {
24+
return { exitCode: 0, stdout: `${STUBBED_APPLE_TOOLCHAIN.sdkVersion}\n`, stderr: '' };
25+
}
26+
throw new Error(`Unexpected Apple toolchain probe: ${command} ${args.join(' ')}`);
27+
}
28+
29+
/**
30+
* Answers the runner cache's toolchain probes from a fixed toolchain, so cases
31+
* that key the cache neither read the host's Xcode nor depend on one existing.
32+
* Returns the mock so a case can make a probe fail.
33+
*/
34+
export function stubAppleToolchainProbes(): ReturnType<typeof vi.fn> {
35+
const runCmdSync = vi.fn(appleToolchainProbeResult);
36+
beforeEach(() => {
37+
runCmdSync.mockImplementation(appleToolchainProbeResult);
38+
appleRunnerTestHost.update({ runCmdSync });
39+
});
40+
return runCmdSync;
41+
}

‎packages/platform-apple/src/runner/__tests__/runner-cache-metadata.test.ts‎

Lines changed: 232 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,19 @@
1-
import fs from 'node:fs';
2-
import path from 'node:path';
3-
import { onTestFinished, test } from 'vitest';
1+
import { expect, test } from 'vitest';
42
import assert from 'node:assert/strict';
3+
import { AppError } from '@agent-device/kernel/errors';
54
import { IOS_DEVICE, IOS_SIMULATOR, MACOS_DEVICE } from './device-fixtures.ts';
65
import {
6+
diffComparableRunnerCacheMetadata,
77
resolveRunnerBundleBuildSettings,
88
resolveRunnerMaxConcurrentDestinationsFlag,
99
resolveRunnerSigningBuildSettings,
1010
resolveRunnerPerformanceBuildSettings,
1111
resolveRunnerSandboxBuildArgs,
1212
resolveExpectedRunnerCacheMetadata,
1313
} from '../runner-cache-metadata.ts';
14-
import { mkdtempForTestSync } from './tmp-dir.ts';
14+
import { appleToolchainProbeResult, stubAppleToolchainProbes } from './apple-toolchain-fixtures.ts';
15+
16+
const runCmdSync = stubAppleToolchainProbes();
1517

1618
test('resolveRunnerMaxConcurrentDestinationsFlag uses simulator flag for simulators', () => {
1719
assert.equal(
@@ -40,7 +42,10 @@ test('resolveRunnerSigningBuildSettings returns empty args without env overrides
4042

4143
test('resolveRunnerSigningBuildSettings disables signing for macOS desktop builds', () => {
4244
assert.deepEqual(
43-
resolveRunnerSigningBuildSettings({}, true, { platform: 'apple', appleOs: 'macos' }),
45+
resolveRunnerSigningBuildSettings({}, true, {
46+
platform: 'apple',
47+
appleOs: 'macos',
48+
}),
4449
[
4550
'CODE_SIGNING_ALLOWED=NO',
4651
'CODE_SIGNING_REQUIRED=NO',
@@ -153,83 +158,229 @@ test('resolveRunnerBundleBuildSettings uses AGENT_DEVICE_IOS_BUNDLE_ID when prov
153158
);
154159
});
155160

156-
test('runner cache metadata fingerprints shared snapshot presentation sources', () => {
157-
const root = mkdtempForTestSync('agent-device-runner-cache-fingerprint-');
158-
onTestFinished(() => fs.rmSync(root, { recursive: true, force: true }));
159-
fs.writeFileSync(path.join(root, 'package.json'), JSON.stringify({ version: '0.0.0' }));
160-
fs.mkdirSync(path.join(root, 'apple', 'runner', 'AgentDeviceRunner'), { recursive: true });
161-
fs.mkdirSync(path.join(root, 'apple', 'snapshot-presentation', 'Sources'), { recursive: true });
162-
fs.writeFileSync(
163-
path.join(root, 'apple', 'runner', 'AgentDeviceRunner', 'Runner.swift'),
164-
'runner\n',
165-
);
166-
const sharedSource = path.join(
167-
root,
168-
'apple',
169-
'snapshot-presentation',
170-
'Sources',
171-
'Presentation.swift',
172-
);
173-
fs.writeFileSync(sharedSource, 'shared-one\n');
174-
175-
const before = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR, root).runnerSourceFingerprint;
176-
fs.writeFileSync(sharedSource, 'shared-two\n');
177-
const after = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR, root).runnerSourceFingerprint;
178-
179-
assert.notEqual(after, before);
180-
});
181-
182-
test('runner cache metadata ignores development-only SwiftPM trees but keeps runner unit tests', () => {
183-
const root = mkdtempForTestSync('agent-device-runner-cache-source-roots-');
184-
onTestFinished(() => fs.rmSync(root, { recursive: true, force: true }));
185-
fs.writeFileSync(path.join(root, 'package.json'), JSON.stringify({ version: '0.0.0' }));
186-
187-
const runnerRoot = path.join(root, 'apple', 'runner', 'AgentDeviceRunner');
188-
const runnerUnitTest = path.join(
189-
runnerRoot,
190-
'AgentDeviceRunnerUITests',
191-
'UnitTests',
192-
'Invariant.swift',
193-
);
194-
const sharedRoot = path.join(root, 'apple', 'snapshot-presentation');
195-
fs.mkdirSync(path.dirname(runnerUnitTest), { recursive: true });
196-
fs.mkdirSync(path.join(sharedRoot, 'Sources'), { recursive: true });
197-
fs.writeFileSync(path.join(runnerRoot, 'Runner.swift'), 'runner\n');
198-
fs.writeFileSync(runnerUnitTest, 'unit-one\n');
199-
fs.writeFileSync(path.join(sharedRoot, 'Sources', 'Presentation.swift'), 'shared\n');
200-
201-
for (const directory of [
202-
'Tests',
203-
'SnapshotPresentationConformance',
204-
'.build',
205-
'.swiftpm',
206-
'xcuserdata',
207-
]) {
208-
const file = path.join(sharedRoot, directory, 'Ignored.swift');
209-
fs.mkdirSync(path.dirname(file), { recursive: true });
210-
fs.writeFileSync(file, 'ignored-one\n');
161+
test('metadata diff names only the comparable keys that differ, with expected and actual', () => {
162+
const expected = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR);
163+
const actual = {
164+
...expected,
165+
packageVersion: `${expected.packageVersion}-next`,
166+
xcodeBuildVersion: '17A100',
167+
runnerPerformanceBuildSettings: ['ENABLE_CODE_COVERAGE=YES'],
168+
artifacts: {
169+
xctestrunPath: '/tmp/derived/Runner.xctestrun',
170+
xctestrunMtimeMs: 1,
171+
xctestrunSize: 2,
172+
productPaths: [{ path: '/tmp/derived/Runner.app', mtimeMs: 1, size: 2 }],
173+
},
174+
};
175+
176+
assert.deepEqual(diffComparableRunnerCacheMetadata(expected, actual), [
177+
{
178+
key: 'runnerPerformanceBuildSettings',
179+
expected: JSON.stringify(expected.runnerPerformanceBuildSettings),
180+
actual: '["ENABLE_CODE_COVERAGE=YES"]',
181+
},
182+
{ key: 'xcodeBuildVersion', expected: '"17C52"', actual: '"17A100"' },
183+
]);
184+
});
185+
186+
test('metadata diff reports a key only one side carries as absent', () => {
187+
const expected = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR);
188+
const { sdkBuildVersion: _sdkBuildVersion, ...withoutSdkBuildVersion } = expected;
189+
190+
assert.deepEqual(
191+
diffComparableRunnerCacheMetadata(expected, withoutSdkBuildVersion as typeof expected),
192+
[{ key: 'sdkBuildVersion', expected: '"23C53"', actual: '(absent)' }],
193+
);
194+
});
195+
196+
test('metadata diff is empty for identical metadata', () => {
197+
const expected = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR);
198+
199+
assert.deepEqual(diffComparableRunnerCacheMetadata(expected, { ...expected }), []);
200+
});
201+
202+
test('metadata diff elides an over-long value in the middle so both ends stay comparable', () => {
203+
const expected = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR);
204+
const longSetting = (suffix: string) => [`${'A'.repeat(400)}=${suffix}`];
205+
206+
const [difference] = diffComparableRunnerCacheMetadata(
207+
{ ...expected, runnerBundleBuildSettings: longSetting('one') },
208+
{ ...expected, runnerBundleBuildSettings: longSetting('two') },
209+
);
210+
211+
assert.equal(difference?.key, 'runnerBundleBuildSettings');
212+
assert.ok((difference?.expected.length ?? 0) <= 300);
213+
assert.ok(difference?.expected.startsWith('["AAA'));
214+
assert.ok(difference?.expected.endsWith('=one"]'));
215+
assert.ok(difference?.actual.endsWith('=two"]'));
216+
});
217+
218+
function unavailableProbes(): { probe: string; reason: string }[] {
219+
try {
220+
resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR);
221+
return [];
222+
} catch (error) {
223+
assert.ok(error instanceof AppError);
224+
assert.equal(error.details?.reason, 'apple_toolchain_probe_unavailable');
225+
const probes = error.details?.probes as { probe: string; reason: string }[];
226+
return probes.map(({ probe, reason }) => ({ probe, reason }));
211227
}
228+
}
229+
230+
test('a timed-out probe leaves the toolchain unavailable instead of a comparable value', () => {
231+
runCmdSync.mockImplementation((command: string, args: readonly string[]) => {
232+
if (command === 'xcodebuild') {
233+
throw new AppError('COMMAND_FAILED', 'xcodebuild timed out after 5000ms', {
234+
timeoutMs: 5_000,
235+
});
236+
}
237+
return appleToolchainProbeResult(command, args);
238+
});
239+
240+
assert.deepEqual(unavailableProbes(), [{ probe: 'xcodebuild -version', reason: 'probe_error' }]);
241+
});
242+
243+
test('a failing probe reports its exit status rather than a fabricated SDK version', () => {
244+
runCmdSync.mockImplementation((command: string, args: readonly string[]) =>
245+
command === 'xcrun'
246+
? {
247+
exitCode: 70,
248+
stdout: '',
249+
stderr: 'xcrun: error: SDK cannot be located\n',
250+
}
251+
: appleToolchainProbeResult(command, args),
252+
);
212253

213-
const before = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR, root).runnerSourceFingerprint;
214-
for (const directory of [
215-
'Tests',
216-
'SnapshotPresentationConformance',
217-
'.build',
218-
'.swiftpm',
219-
'xcuserdata',
220-
]) {
221-
fs.writeFileSync(path.join(sharedRoot, directory, 'Ignored.swift'), 'ignored-two\n');
254+
assert.deepEqual(unavailableProbes(), [
255+
{
256+
probe: 'xcrun --sdk iphonesimulator --show-sdk-version',
257+
reason: 'nonzero_exit',
258+
},
259+
{
260+
probe: 'xcrun --sdk iphonesimulator --show-sdk-build-version',
261+
reason: 'nonzero_exit',
262+
},
263+
]);
264+
});
265+
266+
test('unrecognized xcodebuild output is unavailable, not a partially parsed fingerprint', () => {
267+
runCmdSync.mockImplementation((command: string, args: readonly string[]) =>
268+
command === 'xcodebuild'
269+
? {
270+
exitCode: 0,
271+
stdout: 'xcode-select: error: tool not configured\n',
272+
stderr: '',
273+
}
274+
: appleToolchainProbeResult(command, args),
275+
);
276+
277+
assert.deepEqual(unavailableProbes(), [
278+
{ probe: 'xcodebuild -version', reason: 'unparsable_output' },
279+
]);
280+
});
281+
282+
test('an empty probe answer is unavailable rather than an empty cache key field', () => {
283+
runCmdSync.mockImplementation((command: string, args: readonly string[]) =>
284+
command === 'xcrun' && args.includes('--show-sdk-build-version')
285+
? { exitCode: 0, stdout: '\n', stderr: '' }
286+
: appleToolchainProbeResult(command, args),
287+
);
288+
289+
assert.deepEqual(unavailableProbes(), [
290+
{
291+
probe: 'xcrun --sdk iphonesimulator --show-sdk-build-version',
292+
reason: 'empty_output',
293+
},
294+
]);
295+
});
296+
297+
test('an unavailable toolchain fails the cache decision with a retriable typed error', () => {
298+
runCmdSync.mockImplementation(() => {
299+
throw new AppError('COMMAND_FAILED', 'xcodebuild timed out after 5000ms', {});
300+
});
301+
302+
try {
303+
resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR);
304+
assert.fail('expected an unavailable toolchain to fail the cache decision');
305+
} catch (error) {
306+
assert.ok(error instanceof AppError);
307+
assert.equal(error.code, 'COMMAND_FAILED');
308+
assert.equal(error.details?.retriable, true);
309+
expect(error.message).toContain('xcodebuild -version');
310+
expect(String(error.details?.hint)).toContain('xcode-select');
222311
}
223-
const afterIgnoredChanges = resolveExpectedRunnerCacheMetadata(
224-
IOS_SIMULATOR,
225-
root,
226-
).runnerSourceFingerprint;
227-
assert.equal(afterIgnoredChanges, before);
228-
229-
fs.writeFileSync(runnerUnitTest, 'unit-two\n');
230-
const afterRunnerTestChange = resolveExpectedRunnerCacheMetadata(
231-
IOS_SIMULATOR,
232-
root,
233-
).runnerSourceFingerprint;
234-
assert.notEqual(afterRunnerTestChange, afterIgnoredChanges);
312+
});
313+
314+
test('an unavailable probe never reaches cache metadata, and is not memoized as one', () => {
315+
runCmdSync.mockImplementation(() => {
316+
throw new AppError('COMMAND_FAILED', 'xcodebuild timed out after 5000ms', {});
317+
});
318+
expect(() => resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR)).toThrow(
319+
/Could not read the Xcode toolchain versions/,
320+
);
321+
322+
runCmdSync.mockImplementation(appleToolchainProbeResult);
323+
const metadata = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR);
324+
325+
assert.equal(metadata.xcodeVersion, '26.2');
326+
assert.equal(metadata.xcodeBuildVersion, '17C52');
327+
assert.equal(metadata.sdkVersion, '26.2');
328+
assert.equal(metadata.sdkBuildVersion, '23C53');
329+
});
330+
331+
test('a malformed xcodebuild answer is not memoized: the next request re-probes and recovers', () => {
332+
runCmdSync.mockImplementation((command: string, args: readonly string[]) =>
333+
command === 'xcodebuild'
334+
? {
335+
exitCode: 0,
336+
stdout: 'xcode-select: error: tool not configured\n',
337+
stderr: '',
338+
}
339+
: appleToolchainProbeResult(command, args),
340+
);
341+
assert.deepEqual(unavailableProbes(), [
342+
{ probe: 'xcodebuild -version', reason: 'unparsable_output' },
343+
]);
344+
const probeCallsWhileMalformed = runCmdSync.mock.calls.length;
345+
346+
runCmdSync.mockImplementation(appleToolchainProbeResult);
347+
const metadata = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR);
348+
349+
assert.equal(metadata.xcodeVersion, '26.2');
350+
assert.equal(metadata.xcodeBuildVersion, '17C52');
351+
expect(runCmdSync.mock.calls.slice(probeCallsWhileMalformed).map(([command]) => command)).toEqual(
352+
['xcodebuild', 'xcrun', 'xcrun'],
353+
);
354+
});
355+
356+
test('only a complete, parsed toolchain fingerprint is memoized', () => {
357+
runCmdSync.mockImplementation((command: string, args: readonly string[]) =>
358+
command === 'xcrun' && args.includes('--show-sdk-build-version')
359+
? { exitCode: 0, stdout: '\n', stderr: '' }
360+
: appleToolchainProbeResult(command, args),
361+
);
362+
assert.deepEqual(unavailableProbes(), [
363+
{
364+
probe: 'xcrun --sdk iphonesimulator --show-sdk-build-version',
365+
reason: 'empty_output',
366+
},
367+
]);
368+
369+
runCmdSync.mockImplementation(appleToolchainProbeResult);
370+
runCmdSync.mockClear();
371+
const first = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR);
372+
// The healthy xcodebuild answer from the failed round was not kept either: all three re-run.
373+
expect(runCmdSync.mock.calls.map(([command]) => command)).toEqual([
374+
'xcodebuild',
375+
'xcrun',
376+
'xcrun',
377+
]);
378+
379+
runCmdSync.mockClear();
380+
const second = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR);
381+
expect(runCmdSync).not.toHaveBeenCalled();
382+
assert.deepEqual(
383+
[second.xcodeVersion, second.xcodeBuildVersion, second.sdkVersion, second.sdkBuildVersion],
384+
[first.xcodeVersion, first.xcodeBuildVersion, first.sdkVersion, first.sdkBuildVersion],
385+
);
235386
});

0 commit comments

Comments
 (0)