Skip to content

Commit 9cf6410

Browse files
author
Cameron Chisholm
committed
fix(screenshot): require CoreDevice capture evidence for physical crops
1 parent aa9dec2 commit 9cf6410

12 files changed

Lines changed: 182 additions & 17 deletions

File tree

‎packages/contracts/src/interactor-types.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,11 @@ export type ScreenshotOptions = {
7474
* rendered the pixels in, so a landscape value pairs with a landscape image; it is absent when the
7575
* owner has no rotation reading it can stand behind for this capture.
7676
*/
77-
export type ScreenshotCaptureFacts = Readonly<{ displayRotation?: DeviceRotation }>;
77+
export type ScreenshotCaptureFacts = Readonly<{
78+
displayRotation?: DeviceRotation;
79+
/** Actual physical iOS capture route, including any fallback from CoreDevice to XCTest. */
80+
iosPhysicalCaptureBackend?: 'coredevice' | 'xctest';
81+
}>;
7882

7983
export type ElementSelectorKey = 'id' | 'label' | 'text' | 'value';
8084

‎packages/daemon/src/execution/__tests__/screenshot-crop-target.test.ts‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,14 @@ type CropTargetDevice = Readonly<{
3030
}>;
3131

3232
const IPADOS_DEVICE: DeviceInfo = { ...IPADOS_SIMULATOR, kind: 'device' };
33+
const IOS_DEVICE_COREDEVICE: DeviceInfo = {
34+
...IOS_DEVICE,
35+
iosPhysicalDeviceBackend: 'coredevice',
36+
};
37+
const IOS_DEVICE_XCTEST: DeviceInfo = {
38+
...IOS_DEVICE,
39+
iosPhysicalDeviceBackend: 'xctest',
40+
};
3341

3442
/** The cells with live pixel-identity evidence; every other cell stays rejected. */
3543
const ACCEPTED_CROP_TARGETS: readonly CropTargetDevice['target'][] = [
@@ -45,6 +53,7 @@ const CROP_TARGET_DEVICES: readonly CropTargetDevice[] = [
4553
{ target: 'android-device', device: ANDROID_DEVICE, surface: undefined },
4654
{ target: 'macos-app-window', device: MACOS_DEVICE, surface: 'app' },
4755
{ target: 'ios-physical', device: IOS_DEVICE, surface: undefined },
56+
{ target: 'ios-physical-xctest', device: IOS_DEVICE_XCTEST, surface: undefined },
4857
{ target: 'ipados-physical', device: IPADOS_DEVICE, surface: undefined },
4958
{ target: 'macos-helper', device: MACOS_DEVICE, surface: 'desktop' },
5059
{ target: 'web', device: WEB_DESKTOP_DEVICE, surface: undefined },
@@ -87,6 +96,40 @@ test('a physical iPad has its own cell, and an iPad simulator stays in the simul
8796
expect(classifyScreenshotCropTarget(IPADOS_SIMULATOR, undefined)).toBe('ios-simulator');
8897
});
8998

99+
test('a physical iPhone on CoreDevice passes the policy', () => {
100+
expect(classifyScreenshotCropTarget(IOS_DEVICE_COREDEVICE, undefined)).toBe('ios-physical');
101+
expect(() =>
102+
assertScreenshotCropPolicy({
103+
device: IOS_DEVICE_COREDEVICE,
104+
surface: undefined,
105+
cropOn: 'label="Continue"',
106+
overlayRefs: false,
107+
fullscreen: false,
108+
}),
109+
).not.toThrow();
110+
});
111+
112+
test('a physical iPhone on the XCTest backend is refused with the pending-evidence reason, naming its own cell', () => {
113+
expect(() =>
114+
assertScreenshotCropPolicy({
115+
device: IOS_DEVICE_XCTEST,
116+
surface: undefined,
117+
cropOn: 'label="Continue"',
118+
overlayRefs: false,
119+
fullscreen: false,
120+
}),
121+
).toThrow(
122+
expect.objectContaining({
123+
code: 'UNSUPPORTED_OPERATION',
124+
message: 'screenshot --crop-on is not accepted on ios-physical-xctest targets',
125+
details: {
126+
reason: SCREENSHOT_CROP_REASONS.targetNotAccepted,
127+
rejectionReason: SCREENSHOT_CROP_REASONS.pendingPixelIdentityEvidence,
128+
},
129+
}),
130+
);
131+
});
132+
90133
test('a physical iPad is refused with the pending-evidence reason, naming its own cell', () => {
91134
expect(() =>
92135
assertScreenshotCropPolicy({
@@ -205,7 +248,7 @@ test('a matrix-rejected target is refused with the pending-evidence reason', ()
205248
});
206249

207250
test('an accepted target with a valid selector passes the policy', () => {
208-
for (const device of [IOS_SIMULATOR, ANDROID_EMULATOR, IOS_DEVICE]) {
251+
for (const device of [IOS_SIMULATOR, ANDROID_EMULATOR]) {
209252
expect(() =>
210253
assertScreenshotCropPolicy({ device, surface: undefined, ...POLICY_BASE }),
211254
).not.toThrow();

‎packages/daemon/src/execution/screenshot-crop-target.ts‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import type { ScreenshotCaptureFacts } from '@agent-device/contracts/screenshot-runtime';
12
import {
23
SCREENSHOT_CROP_REASONS,
34
type ScreenshotCropReason,
@@ -19,6 +20,7 @@ type CropTarget =
1920
| 'android-device'
2021
| 'macos-app-window'
2122
| 'ios-physical'
23+
| 'ios-physical-xctest'
2224
| 'ipados-physical'
2325
| 'macos-helper'
2426
| 'web'
@@ -48,6 +50,7 @@ export const SCREENSHOT_CROP_TARGET_CELLS: readonly CropTargetCell[] = [
4850
{ target: 'android-device', status: 'rejected', rejectionReason: PENDING },
4951
{ target: 'macos-app-window', status: 'rejected', rejectionReason: PENDING },
5052
{ target: 'ios-physical', status: 'accepted' },
53+
{ target: 'ios-physical-xctest', status: 'rejected', rejectionReason: PENDING },
5154
{ target: 'ipados-physical', status: 'rejected', rejectionReason: PENDING },
5255
{ target: 'macos-helper', status: 'rejected', rejectionReason: PENDING },
5356
{ target: 'web', status: 'rejected', rejectionReason: PENDING },
@@ -85,7 +88,8 @@ function classifyAppleCropTarget(
8588
const appleOs = resolveDeviceAppleOs(device);
8689
switch (appleOs) {
8790
case 'ios':
88-
return device.kind === 'device' ? 'ios-physical' : 'ios-simulator';
91+
if (device.kind !== 'device') return 'ios-simulator';
92+
return device.iosPhysicalDeviceBackend === 'xctest' ? 'ios-physical-xctest' : 'ios-physical';
8993
case 'ipados':
9094
// Only a physical iPhone has crop evidence. An iPad's largest viewport root can be the app
9195
// window rather than the screen, so its physical capture frame is not yet checked.
@@ -152,3 +156,16 @@ export function assertScreenshotCropPolicy(params: {
152156
throw cropRefusal(target, cell?.status === 'rejected' ? cell.rejectionReason : undefined);
153157
}
154158
}
159+
160+
/** Physical iPhone crop evidence applies to the image producer, not just the selected backend. */
161+
export function assertScreenshotCropCapture(
162+
device: DeviceInfo,
163+
facts: ScreenshotCaptureFacts,
164+
): void {
165+
if (
166+
classifyScreenshotCropTarget(device, undefined) === 'ios-physical' &&
167+
facts.iosPhysicalCaptureBackend !== 'coredevice'
168+
) {
169+
throw cropRefusal('ios-physical-non-coredevice-capture', PENDING);
170+
}
171+
}

‎packages/daemon/src/execution/screenshot-runtime-execution.test.ts‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,12 @@ import path from 'node:path';
55
import { expect, test } from 'vitest';
66
import { SCREENSHOT_CROP_REASONS } from '@agent-device/contracts/capture';
77
import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
8-
import { ANDROID_EMULATOR, IOS_SIMULATOR, mkdtempForTestSync } from '@agent-device/test-kit';
8+
import {
9+
ANDROID_EMULATOR,
10+
IOS_DEVICE,
11+
IOS_SIMULATOR,
12+
mkdtempForTestSync,
13+
} from '@agent-device/test-kit';
914
import { makeSession } from '../__tests__/support/session-factories.ts';
1015
import type { GenericPlatformExecutionParams } from '../core/generic-leaf-execution.ts';
1116
import { resolveScreenshotGenericExecution } from './screenshot-runtime.ts';
@@ -428,3 +433,49 @@ test('the result record omits the display rotation when the capture reported non
428433

429434
expect(await executeResult(resolved, session, req)).not.toHaveProperty('displayRotation');
430435
});
436+
437+
for (const backend of ['coredevice', 'xctest', undefined] as const) {
438+
for (const crop of [false, true]) {
439+
test(`physical iPhone capture backend=${backend}, crop=${crop} enforces capture evidence`, async () => {
440+
const fixture = screenshotRuntimeFixture({
441+
onCapture: ({ outPath }) => {
442+
writeSolidPng(outPath);
443+
return backend ? { iosPhysicalCaptureBackend: backend } : {};
444+
},
445+
snapshotResult: () => ({ nodes: CROP_TREE, backend: 'xctest', producer: 'apple-runner' }),
446+
});
447+
const session = makeSession('iphone', { device: IOS_DEVICE });
448+
const outPath = path.join(mkdtempForTestSync('physical-crop'), 'screen.png');
449+
const req = screenshotRequest({
450+
positionals: [outPath],
451+
flags: crop ? { screenshotCropOn: 'label="Save"' } : {},
452+
});
453+
const resolved = await resolveScreenshotGenericExecution({
454+
hostCwd: process.cwd(),
455+
createCommandSurface: testCreateCommandSurface,
456+
platformServices: daemonPlatformServicesFixture(),
457+
req,
458+
session,
459+
inspectFacts: fixture.inspectFacts,
460+
bindDevice: fixture.bindDevice,
461+
});
462+
if (crop && backend !== 'coredevice') {
463+
await expect(executeResult(resolved, session, req)).rejects.toMatchObject({
464+
code: 'UNSUPPORTED_OPERATION',
465+
message:
466+
'screenshot --crop-on is not accepted on ios-physical-non-coredevice-capture targets',
467+
details: {
468+
reason: SCREENSHOT_CROP_REASONS.targetNotAccepted,
469+
rejectionReason: SCREENSHOT_CROP_REASONS.pendingPixelIdentityEvidence,
470+
},
471+
});
472+
expect(fixture.captureSnapshot).not.toHaveBeenCalled();
473+
} else {
474+
const result = await executeResult(resolved, session, req);
475+
expect(result).toMatchObject(crop ? { width: 40, height: 20 } : { width: 100, height: 50 });
476+
expect(fixture.captureSnapshot).toHaveBeenCalledTimes(crop ? 1 : 0);
477+
}
478+
expect(fixture.captureScreenshot).toHaveBeenCalledTimes(1);
479+
});
480+
}
481+
}

‎packages/daemon/src/execution/screenshot-runtime.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,10 @@ import type {
3030
ResolvedGenericExecution,
3131
} from '../core/generic-leaf-execution.ts';
3232
import { createReadonlyRuntimeSessionStore } from '../core/runtime-session.ts';
33-
import { assertScreenshotCropPolicy } from './screenshot-crop-target.ts';
33+
import {
34+
assertScreenshotCropCapture,
35+
assertScreenshotCropPolicy,
36+
} from './screenshot-crop-target.ts';
3437
import { buildScreenshotCropWarnings, cropScreenshotToSelector } from './screenshot-crop.ts';
3538
import { annotateScreenshotWithRefs } from '@agent-device/capture-kit/screenshot-overlay';
3639
import {
@@ -373,6 +376,7 @@ function createBoundScreenshotBackend(
373376
execution: screenshotExecutionFromContext(dispatchContext),
374377
});
375378
if (crop) {
379+
assertScreenshotCropCapture(session.device, facts);
376380
const outcome = await cropScreenshotToSelector({
377381
device: session.device,
378382
session,

‎packages/platform-apple/src/core/__tests__/physical-device-control.test.ts‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ test('default interactor screenshots stay in-band without invoking devicectl', a
9595
const outPath = path.join(await mkdtempForTest('agent-device-xctest-screenshot-'), 'screen.png');
9696
const toolCalls: string[][] = [];
9797
try {
98-
await withAppleToolProvider(
98+
const facts = await withAppleToolProvider(
9999
createLocalAppleToolProvider({
100100
runCommand: async (_cmd, args) => {
101101
toolCalls.push(args);
@@ -120,6 +120,7 @@ test('default interactor screenshots stay in-band without invoking devicectl', a
120120
),
121121
);
122122

123+
assert.deepEqual(facts, { iosPhysicalCaptureBackend: 'xctest' });
123124
assert.equal(await fs.readFile(outPath, 'utf8'), 'png-bytes');
124125
assert.equal(
125126
toolCalls.some((args) => args[0] === 'devicectl'),
@@ -136,3 +137,41 @@ test('CoreDevice-only app inventory fails with targeted XCTest guidance', async
136137
/App inventory is unavailable on this XCTest-backed physical iOS device/,
137138
);
138139
});
140+
141+
for (const fallback of [false, true]) {
142+
test(`physical interactor reports the actual screenshot backend after fallback=${fallback}`, async () => {
143+
const outPath = path.join(await mkdtempForTest('agent-device-capture-backend-'), 'screen.png');
144+
let runnerCalls = 0;
145+
const facts = await withAppleToolProvider(
146+
createLocalAppleToolProvider({
147+
runCommand: async (_cmd, args) => {
148+
assert.deepEqual(args.slice(0, 4), ['devicectl', 'device', 'capture', 'screenshot']);
149+
if (fallback)
150+
return { exitCode: 1, stdout: '', stderr: "error: Unknown option '--device'" };
151+
await fs.writeFile(outPath, 'coredevice-png');
152+
return { exitCode: 0, stdout: '', stderr: '' };
153+
},
154+
}),
155+
async () =>
156+
await withAppleRunnerProvider(
157+
async () => {
158+
runnerCalls += 1;
159+
return { imageBase64: Buffer.from('runner-png').toString('base64') };
160+
},
161+
{ deviceId: IOS_DEVICE.id },
162+
async () =>
163+
await createAppleInteractor(
164+
IOS_DEVICE,
165+
{},
166+
{
167+
env: process.env,
168+
cwd: process.cwd(),
169+
},
170+
).screenshot(outPath),
171+
),
172+
);
173+
assert.deepEqual(facts, { iosPhysicalCaptureBackend: fallback ? 'xctest' : 'coredevice' });
174+
assert.equal(runnerCalls, fallback ? 1 : 0);
175+
assert.equal(await fs.readFile(outPath, 'utf8'), fallback ? 'runner-png' : 'coredevice-png');
176+
});
177+
}

‎packages/platform-apple/src/core/physical-device-control.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import type { ScreenshotCaptureFacts } from '@agent-device/contracts/interactor-types';
12
import type { DeviceInfo } from '@agent-device/kernel/device';
23
import type { EnvMap } from '@agent-device/kernel/source-value';
34
import { AppError } from '@agent-device/kernel/errors';
@@ -61,7 +62,7 @@ export type IosPhysicalDeviceControl = IosPhysicalDeviceRunnerControl & {
6162
device: DeviceInfo,
6263
outPath: string,
6364
options: IosPhysicalDeviceScreenshotOptions,
64-
): Promise<void>;
65+
): Promise<ScreenshotCaptureFacts>;
6566
copyRunnerFile(
6667
device: DeviceInfo,
6768
remotePath: string,

‎packages/platform-apple/src/core/physical-device-screenshot.ts‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import type { ScreenshotCaptureFacts } from '@agent-device/contracts/interactor-types';
12
import type { DeviceInfo } from '@agent-device/kernel/device';
23
import type { EnvMap } from '@agent-device/kernel/source-value';
34
import { AppError } from '@agent-device/kernel/errors';
@@ -22,7 +23,7 @@ export async function captureCoreDeviceScreenshot(
2223
device: DeviceInfo,
2324
outPath: string,
2425
options: IosPhysicalDeviceScreenshotOptions,
25-
): Promise<void> {
26+
): Promise<ScreenshotCaptureFacts> {
2627
if (!options.preferRunner) {
2728
try {
2829
await runIosDevicectl(
@@ -32,27 +33,30 @@ export async function captureCoreDeviceScreenshot(
3233
deviceId: device.id,
3334
},
3435
);
35-
return;
36+
return { iosPhysicalCaptureBackend: 'coredevice' };
3637
} catch (error) {
3738
if (!shouldFallbackToRunnerForIosScreenshot(error)) throw error;
3839
emitPhysicalScreenshotFallbackDiagnostic(device, error);
3940
}
4041
}
4142

4243
const result = await runRunnerScreenshot(device, options, false);
43-
if (await writeInlineRunnerScreenshot(result, outPath)) return;
44+
if (await writeInlineRunnerScreenshot(result, outPath))
45+
return { iosPhysicalCaptureBackend: 'xctest' };
4446

4547
const remoteFileName = readRunnerScreenshotPath(result);
4648
await copyCoreDeviceRunnerFile(device, remoteFileName, outPath, options.env);
49+
return { iosPhysicalCaptureBackend: 'xctest' };
4750
}
4851

4952
export async function captureXctestDeviceScreenshot(
5053
device: DeviceInfo,
5154
outPath: string,
5255
options: IosPhysicalDeviceScreenshotOptions,
53-
): Promise<void> {
56+
): Promise<ScreenshotCaptureFacts> {
5457
const result = await runRunnerScreenshot(device, options, true);
55-
if (await writeInlineRunnerScreenshot(result, outPath)) return;
58+
if (await writeInlineRunnerScreenshot(result, outPath))
59+
return { iosPhysicalCaptureBackend: 'xctest' };
5660
throw new AppError(
5761
'COMMAND_FAILED',
5862
'Failed to capture iOS screenshot: XCTest runner returned no inline image',

‎packages/platform-apple/src/core/screenshot.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,14 +117,13 @@ export async function screenshotIos(
117117
return await captureSimulatorScreenshotWithFallback(device, outPath, options);
118118
}
119119

120-
await resolveIosPhysicalDeviceControl(device).captureScreenshot(device, outPath, {
120+
return await resolveIosPhysicalDeviceControl(device).captureScreenshot(device, outPath, {
121121
appBundleId: options.appBundleId,
122122
fullscreen: options.fullscreen,
123123
runnerOptions: options.runnerOptions,
124124
runRunnerCommand: runAppleRunnerCommand,
125125
env: options.env,
126126
});
127-
return {};
128127
}
129128

130129
export async function captureSimulatorScreenshotWithFallback(

‎src/commands/capture/screenshot.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ export const screenshotCommandFacet = defineCommandFacet({
7474
text: {
7575
summary: 'Capture a screenshot',
7676
cliDetail:
77-
"Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on <selector> to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators, physical iPhones, and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent.",
77+
"Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on <selector> to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators, Android emulators, and CoreDevice physical iPhones), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Physical iPhone crop verification covers portrait CoreDevice/devicectl captures only. XCTest, runner fallback, and physical iPad crops are refused; landscape iPhone crop is unverified. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent.",
7878
},
7979
metadata: screenshotCommandMetadata,
8080
run: (client, input) => client.capture.screenshot(input),

0 commit comments

Comments
 (0)