Skip to content

Commit a371ad2

Browse files
committed
fix(ios-runner): let the device speak only where nothing matched
Keying the enrichment on `build_failed_unclassified` read the absence of a row as a positive claim: a row that deliberately names no cause publishes the same reason, so a just-rebooted phone's disk-image state would have been published over a conflicting-settings failure. The classifier now reports whether a row matched, the catch that classifies carries that answer on `details.startupRuleMatched`, and the enrichment reads it. The enrichment also moves out of the build catch into the session's startup catch, which already holds the device's states and sees every step from the artifact to a runner that answers. A warm derived cache that fails at install and an external xctestrun that never launches now carry `details.developerDiskImage` too, which the build-catch-only version could not reach.
1 parent 32c5914 commit a371ad2

7 files changed

Lines changed: 284 additions & 103 deletions

File tree

‎packages/platform-apple/src/runner/__tests__/runner-session-readiness.test.ts‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import {
1616
} from './runner-session-fixtures.ts';
1717
import { mkdtempForTestSync } from './tmp-dir.ts';
1818
import { createLocalAppleToolProvider, withAppleToolProvider } from '../../core/tool-provider.ts';
19+
import { IOS_DEVICE_DEVELOPER_DISK_IMAGE_HINT } from '../../core/devicectl.ts';
1920

2021
const {
2122
mockAcquireXcodebuildSimulatorSetRedirect,
@@ -601,3 +602,109 @@ function jsonOutputPathOf(args: string[]): string | undefined {
601602
const index = args.indexOf('--json-output');
602603
return index >= 0 ? args[index + 1] : undefined;
603604
}
605+
606+
/**
607+
* The startup catch — not the build catch — is where the device's own answer attaches (#2690 review):
608+
* a cold build, a warm derived cache that fails at install, and an external xctestrun that never
609+
* launches are different steps, and a caller told "developer disk image" should not have to know which
610+
* one this run happened to take. What each step throws below is the shape that step publishes; what is
611+
* under test is what the session adds on the way out.
612+
*/
613+
test('a build that named no cause on a device with its image down gets the device answer', async () => {
614+
const device = { ...IOS_DEVICE, id: 'runner-session-image-down-build' };
615+
mockEnsureXctestrunArtifact.mockRejectedValue(
616+
new AppError('COMMAND_FAILED', 'xcodebuild build-for-testing failed', {
617+
reason: 'build_failed_unclassified',
618+
startupRuleMatched: false,
619+
details: { stdout: "error: cannot find 'AgentDeviceRunnerCommand' in scope\n" },
620+
}),
621+
);
622+
623+
const error = await expectStartupFailure(device, deviceDetailsPayload('enabled', false));
624+
625+
assert.equal(error.details?.reason, 'device_developer_disk_image_unavailable');
626+
assert.equal(error.details?.developerDiskImage, 'unavailable');
627+
assert.ok(String(error.details?.hint).includes(IOS_DEVICE_DEVELOPER_DISK_IMAGE_HINT));
628+
});
629+
630+
test('a warm cache that fails at launch still carries what the device said', async () => {
631+
// The case the build-catch-only version missed: nothing had to compile, so the device's answer was
632+
// never attached anywhere, and an install that cannot start the runner said only "build failed".
633+
const device = { ...IOS_DEVICE, id: 'runner-session-image-down-launch' };
634+
mockEnsureXctestrunArtifact.mockResolvedValue({
635+
xctestrunPath: '/tmp/base-runner.xctestrun',
636+
derived: '/tmp/derived',
637+
cache: 'exact',
638+
artifact: 'valid',
639+
buildMs: 0,
640+
xctestrunPathSource: 'manifest',
641+
});
642+
mockRunCmdBackground.mockImplementation(() => {
643+
throw new AppError('COMMAND_FAILED', 'xcodebuild test-without-building exited unexpectedly');
644+
});
645+
646+
const error = await expectStartupFailure(device, deviceDetailsPayload('enabled', false));
647+
648+
assert.equal(error.details?.reason, 'device_developer_disk_image_unavailable');
649+
assert.equal(error.details?.developerDiskImage, 'unavailable');
650+
});
651+
652+
test('a device whose image is available claims nothing for a failure it did not cause', async () => {
653+
const device = { ...IOS_DEVICE, id: 'runner-session-image-available' };
654+
mockEnsureXctestrunArtifact.mockRejectedValue(
655+
new AppError('COMMAND_FAILED', 'xcodebuild build-for-testing failed', {
656+
reason: 'build_failed_unclassified',
657+
startupRuleMatched: false,
658+
details: { stdout: "error: cannot find 'AgentDeviceRunnerCommand' in scope\n" },
659+
}),
660+
);
661+
662+
const error = await expectStartupFailure(device, deviceDetailsPayload('enabled', true));
663+
664+
assert.equal(error.details?.reason, 'build_failed_unclassified');
665+
assert.equal(error.details?.developerDiskImage, 'available');
666+
});
667+
668+
/** `devicectl device info details` for a phone that is awake, connected, and reporting both states. */
669+
function deviceDetailsPayload(
670+
developerModeStatus: 'enabled' | 'disabled',
671+
ddiServicesAvailable: boolean,
672+
): string {
673+
return JSON.stringify({
674+
info: { outcome: 'success' },
675+
result: {
676+
deviceProperties: {
677+
developerModeStatus,
678+
ddiServicesAvailable,
679+
bootState: 'booted',
680+
},
681+
connectionProperties: { tunnelState: 'connected' },
682+
},
683+
});
684+
}
685+
686+
/** Starts a session against a device whose `devicectl` answers with `payload`, and returns the failure. */
687+
async function expectStartupFailure(device: typeof IOS_DEVICE, payload: string): Promise<AppError> {
688+
let caught: unknown;
689+
await assert.rejects(
690+
() =>
691+
withAppleToolProvider(
692+
createLocalAppleToolProvider({
693+
runCommand: async (_cmd: string, args: string[]) => {
694+
const outputPath = jsonOutputPathOf(args);
695+
if (outputPath) {
696+
fs.writeFileSync(outputPath, payload);
697+
}
698+
return { exitCode: 0, stdout: '', stderr: '' };
699+
},
700+
}),
701+
() => ensureRunnerSession(device, {}),
702+
),
703+
(error: unknown) => {
704+
caught = error;
705+
return true;
706+
},
707+
);
708+
assert.ok(caught instanceof AppError, 'startup must fail with an AppError');
709+
return caught;
710+
}

‎packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,17 @@ export const RUNNER_STARTUP_FAILURE_FIXTURES: readonly RunnerStartupFailureFixtu
251251
deviceReport: { developerMode: 'enabled', developerDiskImage: 'unavailable' },
252252
note: 'The corroborated pairing (#2683 review): a build that names no cause, on a phone core read directly as reporting its image down. Naming the image beats cache-recovery advice; the state also travels as details.developerDiskImage.',
253253
},
254+
{
255+
id: 'conflicting-settings-on-device-with-image-down',
256+
reason: 'build_failed_unclassified',
257+
site: 'build-for-testing',
258+
xcodeVersion: UNOBSERVED,
259+
provenance: 'invented-shape',
260+
output:
261+
"error: \"AgentDeviceRunner\" has conflicting provisioning settings (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n",
262+
deviceReport: { developerMode: 'enabled', developerDiskImage: 'unavailable' },
263+
note: 'The pairing that made the enrichment key on "did a row match" rather than on the unclassified reason (#2690 review): a just-rebooted phone reports its image down while the failure is a settings disagreement a row already looked at and declined to name. The row answer wins and the cache-recovery hint stays; the image state still rides along as a detail.',
264+
},
254265
{
255266
id: 'team-id-failure-on-device-with-image-down',
256267
reason: 'signing_no_development_team',

‎packages/platform-apple/src/runner/__tests__/runner-startup-failure-reasons.test.ts‎

Lines changed: 40 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import { appleRunnerTestHost } from '../test-host.ts';
1212
import type { ExecResult } from '@agent-device/host-kit/command';
1313
import { createRunnerPhaseBudget, ensureXctestrunArtifact } from '../runner-xctestrun.ts';
1414
import {
15+
enrichRunnerStartupFailureWithDeviceStates,
1516
RUNNER_DEVICE_READINESS_FAILURE_REASONS,
1617
RUNNER_ERROR_RULES,
1718
classifyRunnerStartupFailure,
@@ -47,23 +48,23 @@ import { mkdtempForTestSync } from './tmp-dir.ts';
4748

4849
const CACHE_RECOVERY_HINT = /clean:xcuitest|apple-runner\/derived/;
4950

50-
/** The device remedies are prose with parentheses in it, so they are matched as text, not as syntax. */
51-
function escapeRegExp(text: string): RegExp {
52-
return new RegExp(text.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`));
53-
}
54-
55-
const HINT_FOR_REASON: Record<RunnerStartupFailureReason, RegExp> = {
56-
bundle_identifier_already_registered: /AGENT_DEVICE_IOS_BUNDLE_ID/,
57-
signing_no_development_team: /AGENT_DEVICE_IOS_TEAM_ID/,
58-
signing_provisioning_profile_missing: /AGENT_DEVICE_IOS_PROVISIONING_PROFILE/,
59-
signing_unspecified: /Automatic Signing/,
60-
devtools_security_developer_mode_disabled: /DevToolsSecurity -enable/,
51+
/**
52+
* The phrase each reason's advice has to contain. Kept as text rather than as syntax because two of
53+
* them are quotations from `core/devicectl.ts`, and a fifth escaping helper for a prose remedy with
54+
* parentheses in it is not this suite's job.
55+
*/
56+
const HINT_FOR_REASON: Record<RunnerStartupFailureReason, string> = {
57+
bundle_identifier_already_registered: 'AGENT_DEVICE_IOS_BUNDLE_ID',
58+
signing_no_development_team: 'AGENT_DEVICE_IOS_TEAM_ID',
59+
signing_provisioning_profile_missing: 'AGENT_DEVICE_IOS_PROVISIONING_PROFILE',
60+
signing_unspecified: 'Automatic Signing',
61+
devtools_security_developer_mode_disabled: 'DevToolsSecurity -enable',
6162
// Both device remedies are owned by `core/devicectl.ts` and travel on the device report, so this
6263
// table quotes them instead of restating them; `runner-device-readiness.test.ts` is where the
6364
// preflight publishing them is asserted.
64-
device_developer_mode_disabled: escapeRegExp(IOS_DEVICE_DEVELOPER_MODE_OFF_HINT),
65-
device_developer_disk_image_unavailable: escapeRegExp(IOS_DEVICE_DEVELOPER_DISK_IMAGE_HINT),
66-
build_failed_unclassified: CACHE_RECOVERY_HINT,
65+
device_developer_mode_disabled: IOS_DEVICE_DEVELOPER_MODE_OFF_HINT,
66+
device_developer_disk_image_unavailable: IOS_DEVICE_DEVELOPER_DISK_IMAGE_HINT,
67+
build_failed_unclassified: 'clean:xcuitest',
6768
};
6869

6970
const runCmdSync = vi.fn();
@@ -129,7 +130,10 @@ function assertFailureEnvelope(
129130
assert.equal(envelope.code, 'COMMAND_FAILED');
130131
assert.equal(envelope.message, 'xcodebuild build-for-testing failed');
131132
assert.equal(envelope.details?.reason, fixture.reason);
132-
assert.match(String(envelope.hint), HINT_FOR_REASON[fixture.reason]);
133+
assert.ok(
134+
String(envelope.hint).includes(HINT_FOR_REASON[fixture.reason]),
135+
`the ${fixture.reason} hint must carry "${HINT_FOR_REASON[fixture.reason]}"`,
136+
);
133137
// No `logPath` was handed to `normalizeError`: the top-level value can only be the one the
134138
// build catch wrote into the error it throws.
135139
assert.equal(envelope.logPath, logPath);
@@ -264,17 +268,35 @@ test('a conflicting-settings failure is not answered with missing-profile advice
264268
assert.doesNotMatch(String(envelope.hint), /AGENT_DEVICE_IOS_PROVISIONING_PROFILE/);
265269
});
266270

267-
/** Drives a recorded fixture through the real build catch and normalizes what it threw. */
271+
/**
272+
* Drives a recorded fixture through the two steps a real startup runs in order: the build catch turns
273+
* the tool's output into a typed reason, and the session's startup catch hands that failure to the
274+
* device enrichment step (#2690 review). Both are the production functions; nothing here re-implements
275+
* either.
276+
*/
268277
async function driveBuildFailure(fixture: RunnerStartupFailureFixture): Promise<NormalizedError> {
269-
return normalizeThrown(await runBuildCatch(() => buildForTestingExecFailure(fixture), fixture));
278+
const thrown = await runBuildCatch(() => buildForTestingExecFailure(fixture));
279+
return normalizeThrown(
280+
enrichRunnerStartupFailureWithDeviceStates(thrown, deviceStatesOf(fixture)),
281+
);
282+
}
283+
284+
/** The states `preflightIosRunnerDeviceReadiness` would have handed the startup for this fixture. */
285+
function deviceStatesOf(fixture: RunnerStartupFailureFixture | undefined) {
286+
if (!fixture?.deviceReport) return undefined;
287+
return {
288+
developerMode: fixture.deviceReport.developerMode,
289+
developerDiskImage: fixture.deviceReport.developerDiskImage,
290+
developerDiskImageHint: IOS_DEVICE_DEVELOPER_DISK_IMAGE_HINT,
291+
};
270292
}
271293

272294
/** Drives a hand-built rejection through the same real build catch. */
273295
async function driveBuildRejection(rejection: unknown): Promise<NormalizedError> {
274296
return normalizeThrown(await runBuildCatch(() => rejection));
275297
}
276298

277-
async function runBuildCatch(buildRejection: () => unknown, fixture?: RunnerStartupFailureFixture) {
299+
async function runBuildCatch(buildRejection: () => unknown): Promise<unknown> {
278300
runCmdStreaming.mockReset().mockImplementation(async () => {
279301
throw buildRejection();
280302
});
@@ -285,13 +307,6 @@ async function runBuildCatch(buildRejection: () => unknown, fixture?: RunnerStar
285307
ensureXctestrunArtifact(IOS_DEVICE, {
286308
logPath,
287309
budget: createRunnerPhaseBudget(120_000, undefined),
288-
// The states the startup would have carried from the device, which is the input the
289-
// corroborated disk-image reason reads (#2683).
290-
deviceStates: fixture?.deviceReport && {
291-
developerMode: fixture.deviceReport.developerMode,
292-
developerDiskImage: fixture.deviceReport.developerDiskImage,
293-
developerDiskImageHint: IOS_DEVICE_DEVELOPER_DISK_IMAGE_HINT,
294-
},
295310
}),
296311
(error: unknown) => {
297312
caught = error;

‎packages/platform-apple/src/runner/runner-artifact.ts‎

Lines changed: 7 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,7 @@ import {
1212
} from './host.ts';
1313
import type { ExecBackgroundResult } from '@agent-device/host-kit/command';
1414
import type { DeviceInfo } from '@agent-device/kernel/device';
15-
import {
16-
classifyRunnerStartupFailure,
17-
corroborateRunnerBuildFailureWithDeviceStates,
18-
type IosRunnerDeviceStates,
19-
} from './runner-contract.ts';
15+
import { classifyRunnerStartupFailure } from './runner-contract.ts';
2016
import { logChunk } from './runner-io.ts';
2117
import { withXcodebuildSimulatorSetRedirect } from './runner-device-set.ts';
2218
import {
@@ -86,13 +82,6 @@ type RunnerXctestrunBuildOptions = {
8682
* request cancels both (#2422).
8783
*/
8884
budget?: RunnerPhaseBudget;
89-
/**
90-
* What the device said about itself before this build started (#2683). Nothing here stops the
91-
* build — that decision was the readiness preflight's, and it refuses only for a disabled Developer
92-
* Mode toggle. These states ride onto whatever failure this build produces, so the device's own
93-
* answer surfaces beside the real failure instead of gating a run that this build would clear.
94-
*/
95-
deviceStates?: IosRunnerDeviceStates;
9685
};
9786

9887
export async function ensureXctestrunArtifact(
@@ -530,22 +519,18 @@ async function buildRunnerXctestrun(
530519
error instanceof AppError ? error : new AppError('COMMAND_FAILED', String(error));
531520
// The reason and the hint beside it come from one classifier (#2680), so the reason a caller
532521
// switches on can never disagree with the advice it is handed.
533-
const classified = classifyRunnerStartupFailure(appErr);
534-
// A corroborated device state can name a build that named nothing; it never overwrites one that
535-
// already named its own cause (#2683).
536-
const { reason, hint } = corroborateRunnerBuildFailureWithDeviceStates(
537-
classified,
538-
options.deviceStates,
539-
);
522+
const { reason, hint, matched } = classifyRunnerStartupFailure(appErr);
523+
// `startupRuleMatched` travels with the verdict: this wrapper buries the tool's text a level too
524+
// deep for the rows to read again, and whether a row spoke is not recoverable from the reason
525+
// alone (#2690 review). The device's own state is attached further out, by the startup catch that
526+
// can see this build and the launch after it.
540527
throw new AppError('COMMAND_FAILED', 'xcodebuild build-for-testing failed', {
541528
reason,
542529
error: appErr.message,
543530
details: appErr.details,
544531
logPath: options.logPath,
545532
hint,
546-
...(options.deviceStates
547-
? { developerDiskImage: options.deviceStates.developerDiskImage }
548-
: {}),
533+
startupRuleMatched: matched,
549534
});
550535
}
551536
});

0 commit comments

Comments
 (0)