Skip to content

Commit 6764cfb

Browse files
strzhaoclaude
andcommitted
fix(screenshot): carry burst frames through the client boundary and bound the burst run
Review follow-up on e406a40: - client: normalizeScreenshotCaptureResult and pickScreenshotResultData keep the burst frames, so the SDK default path and the CLI default --json output report every frame path and capture time as the help promises; a frame without a path and a wall clock is dropped and the array stays within the burst ceiling - burst admission models the whole run, not just the waits: count 2-100, and count * (intervalMs + 500 ms capture allowance per frame) within 60 s, so an admitted burst cannot outrun the screenshot request envelope and reset the daemon - screenshot reads count/intervalMs only when the invocation names them (explicit-only in the CLI parser and in the MCP config defaults): a repository count tuned for press and swipe series no longer arms a burst nor refuses --crop-on beside it - burst warnings dedupe across frames and stay within one note per frame - the scripted --pixel-density error text keeps its released wording Co-Authored-By: Claude Code <[email protected]>
1 parent e406a40 commit 6764cfb

11 files changed

Lines changed: 331 additions & 20 deletions

File tree

‎packages/client/src/__tests__/screenshot-result.test.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import assert from 'node:assert/strict';
22
import { test } from 'vitest';
3+
import { SCREENSHOT_BURST_LIMITS } from '@agent-device/contracts/capture';
34
import {
45
normalizeScreenshotCaptureResult,
56
pickScreenshotResultData,
@@ -57,3 +58,57 @@ test('normalizeScreenshotCaptureResult drops a display rotation outside the rota
5758
);
5859
assert.ok(!('displayRotation' in result));
5960
});
61+
62+
// The default daemon payload reaches this normalizer on the SDK's default path and, through
63+
// `pickScreenshotResultData`, on the CLI's default `--json` path. A burst whose frames the
64+
// normalizer dropped would break the result's "reports every frame" promise at both boundaries.
65+
test('normalizeScreenshotCaptureResult carries the burst frames the daemon reported', () => {
66+
const result = normalizeScreenshotCaptureResult(
67+
{
68+
path: '/tmp/burst/page-0001.png',
69+
width: 100,
70+
height: 50,
71+
frames: [
72+
{ path: '/tmp/burst/page-0001.png', capturedAtMs: 1_000, displayRotation: 'portrait' },
73+
{ path: '/tmp/burst/page-0002.png', capturedAtMs: 1_400 },
74+
],
75+
},
76+
'qa',
77+
);
78+
assert.deepEqual(result.frames, [
79+
{ path: '/tmp/burst/page-0001.png', capturedAtMs: 1_000, displayRotation: 'portrait' },
80+
{ path: '/tmp/burst/page-0002.png', capturedAtMs: 1_400 },
81+
]);
82+
assert.equal(result.path, '/tmp/burst/page-0001.png');
83+
assert.deepEqual(result.identifiers, { session: 'qa' });
84+
});
85+
86+
test('normalizeScreenshotCaptureResult keeps only frames with a path and a wall clock, within the burst bound', () => {
87+
const frames: unknown[] = [
88+
...Array.from({ length: SCREENSHOT_BURST_LIMITS.maxCount }, (_, index) => ({
89+
path: `/tmp/burst/f-${index}.png`,
90+
capturedAtMs: index,
91+
})),
92+
{ capturedAtMs: 9_999 },
93+
{ path: '/tmp/burst/extra.png' },
94+
'frame',
95+
];
96+
const result = normalizeScreenshotCaptureResult({ path: '/tmp/burst/f-0.png', frames }, 'qa');
97+
assert.equal(result.frames?.length, SCREENSHOT_BURST_LIMITS.maxCount);
98+
assert.deepEqual(result.frames?.[0], { path: '/tmp/burst/f-0.png', capturedAtMs: 0 });
99+
});
100+
101+
test('normalizeScreenshotCaptureResult omits empty and absent frames', () => {
102+
assert.ok(!('frames' in normalizeScreenshotCaptureResult({ path: '/tmp/a.png' }, 'qa')));
103+
assert.ok(
104+
!('frames' in normalizeScreenshotCaptureResult({ path: '/tmp/a.png', frames: [] }, 'qa')),
105+
);
106+
});
107+
108+
test('pickScreenshotResultData keeps the frames a normalized result carried', () => {
109+
const frames = [{ path: '/tmp/a-0001.png', capturedAtMs: 1 }];
110+
assert.deepEqual(pickScreenshotResultData({ path: '/tmp/a-0001.png', frames }), {
111+
path: '/tmp/a-0001.png',
112+
frames,
113+
});
114+
});

‎packages/client/src/screenshot-result.ts‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,16 @@
1-
import type { ScreenshotResultData } from '@agent-device/contracts/capture';
1+
import {
2+
SCREENSHOT_BURST_LIMITS,
3+
type ScreenshotBurstFrame,
4+
type ScreenshotResultData,
5+
} from '@agent-device/contracts/capture';
26
import type { CaptureScreenshotResult } from '@agent-device/contracts/client';
37
import { isDeviceRotation } from '@agent-device/contracts/device';
48
import { isRecord, parsePoint, parseRect, readRequiredString } from '@agent-device/kernel/record';
59
import { readResponseWarnings } from '@agent-device/kernel/success-text';
610
import type { ScreenshotOverlayRef } from '@agent-device/kernel/snapshot';
711

812
export function pickScreenshotResultData(value: ScreenshotResultData): ScreenshotResultData {
13+
const frames = readScreenshotBurstFrames(value.frames);
914
return {
1015
...(typeof value.path === 'string' ? { path: value.path } : {}),
1116
...(typeof value.width === 'number' ? { width: value.width } : {}),
@@ -14,6 +19,7 @@ export function pickScreenshotResultData(value: ScreenshotResultData): Screensho
1419
...(typeof value.logicalHeight === 'number' ? { logicalHeight: value.logicalHeight } : {}),
1520
...(typeof value.pixelDensity === 'number' ? { pixelDensity: value.pixelDensity } : {}),
1621
...(isDeviceRotation(value.displayRotation) ? { displayRotation: value.displayRotation } : {}),
22+
...(frames ? { frames } : {}),
1723
...(value.overlayRefs ? { overlayRefs: value.overlayRefs } : {}),
1824
...(value.warnings && value.warnings.length > 0 ? { warnings: value.warnings } : {}),
1925
};
@@ -33,6 +39,7 @@ export function normalizeScreenshotCaptureResult(
3339
logicalHeight: screenshot?.logicalHeight,
3440
pixelDensity: screenshot?.pixelDensity,
3541
...(screenshot?.displayRotation ? { displayRotation: screenshot.displayRotation } : {}),
42+
...(screenshot?.frames ? { frames: screenshot.frames } : {}),
3643
overlayRefs: screenshot?.overlayRefs,
3744
...(screenshot?.warnings ? { warnings: screenshot.warnings } : {}),
3845
identifiers: { session },
@@ -58,11 +65,35 @@ function readScreenshotResultData(value: unknown): ScreenshotResultData | undefi
5865
logicalHeight: readNumberField(value, 'logicalHeight'),
5966
pixelDensity: readNumberField(value, 'pixelDensity'),
6067
displayRotation: isDeviceRotation(value.displayRotation) ? value.displayRotation : undefined,
68+
frames: readScreenshotBurstFrames(value.frames),
6169
overlayRefs: readScreenshotOverlayRefs(value.overlayRefs),
6270
...(warnings !== undefined ? { warnings } : {}),
6371
});
6472
}
6573

74+
/**
75+
* The burst frames a payload may report: one plan admits at most `maxCount` frames, so a longer
76+
* array is truncated, and a frame without a path and a wall clock is not evidence of a capture.
77+
*/
78+
function readScreenshotBurstFrames(value: unknown): ScreenshotBurstFrame[] | undefined {
79+
if (!Array.isArray(value)) return undefined;
80+
const frames = value
81+
.slice(0, SCREENSHOT_BURST_LIMITS.maxCount)
82+
.flatMap((frame) => readScreenshotBurstFrame(frame) ?? []);
83+
return frames.length > 0 ? frames : undefined;
84+
}
85+
86+
function readScreenshotBurstFrame(frame: unknown): ScreenshotBurstFrame | undefined {
87+
if (!isRecord(frame)) return undefined;
88+
if (typeof frame.path !== 'string' || frame.path.length === 0) return undefined;
89+
if (typeof frame.capturedAtMs !== 'number') return undefined;
90+
return {
91+
path: frame.path,
92+
capturedAtMs: frame.capturedAtMs,
93+
...(isDeviceRotation(frame.displayRotation) ? { displayRotation: frame.displayRotation } : {}),
94+
};
95+
}
96+
6697
function readNumberField(value: Record<string, unknown>, key: string): number | undefined {
6798
const field = value[key];
6899
return typeof field === 'number' ? field : undefined;

‎packages/command-registry/src/flag-definitions-action.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [
2222
max: 200,
2323
usageLabel: '--count <n>',
2424
usageDescription:
25-
'Repeat count for press, swipe and gesture tap series; screenshot: burst frame count (2-200) captured sequentially into a numbered file sequence',
25+
'Repeat count for press, swipe and gesture tap series; screenshot: burst frame count (2-100) captured sequentially into a numbered file sequence',
2626
projectConfig: true,
2727
recorded: true,
2828
},

‎packages/command-surface/src/capture/screenshot-options.test.ts‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,7 @@ test('screenshot burst admission bounds the count, the interval, and their pair'
227227
assert.throws(() => validateScreenshotBurst({ count: 1 }), {
228228
message: /--count requires an integer from 2 to /,
229229
});
230-
assert.throws(() => validateScreenshotBurst({ count: 201 }), { code: 'INVALID_ARGS' });
230+
assert.throws(() => validateScreenshotBurst({ count: 101 }), { code: 'INVALID_ARGS' });
231231
assert.throws(() => validateScreenshotBurst({ count: 2.5 }), { code: 'INVALID_ARGS' });
232232
assert.throws(() => validateScreenshotBurst({ count: 2, intervalMs: 50 }), {
233233
message: /--interval-ms requires an integer from 100 to /,
@@ -237,19 +237,28 @@ test('screenshot burst admission bounds the count, the interval, and their pair'
237237
});
238238
});
239239

240-
test('screenshot burst admission refuses crop and overlay combinations and overlong schedules', () => {
240+
test('screenshot burst admission refuses crop and overlay combinations and request-budget overruns', () => {
241241
assert.throws(() => validateScreenshotBurst({ count: 3, cropOn: 'role=button label=Save' }), {
242242
message: /--count cannot be combined with --crop-on or --overlay-refs/,
243243
});
244244
assert.throws(() => validateScreenshotBurst({ count: 3, overlayRefs: true }), {
245245
message: /--count cannot be combined with --crop-on or --overlay-refs/,
246246
});
247247
assert.doesNotThrow(() => validateScreenshotBurst({ count: 3, cropOn: undefined }));
248+
// The run budget covers the waits plus one capture allowance per frame, so the admitted burst
249+
// stays inside the request envelope even when every frame costs its full allowance.
248250
assert.throws(
249-
() => validateScreenshotBurst({ count: 150, intervalMs: 500 }),
250-
/keep the burst within 30000 ms/,
251+
() => validateScreenshotBurst({ count: 100, intervalMs: 200 }),
252+
/keep the burst within 60000 ms/,
251253
);
252-
assert.doesNotThrow(() => validateScreenshotBurst({ count: 200 }));
254+
assert.throws(
255+
() => validateScreenshotBurst({ count: 100, intervalMs: 600 }),
256+
/keep the burst within 60000 ms/,
257+
);
258+
assert.doesNotThrow(() => validateScreenshotBurst({ count: 100 }));
259+
assert.throws(() => validateScreenshotBurst({ count: 150 }), {
260+
message: /--count requires an integer from 2 to 100/,
261+
});
253262
});
254263

255264
test('screenshot burst frame paths number the user sequence in place', () => {
@@ -283,3 +292,10 @@ test('screenshot burst flags round-trip through recorded scripts', () => {
283292
{ code: 'INVALID_ARGS' },
284293
);
285294
});
295+
296+
test('the scripted pixel-density flag keeps its released error text', () => {
297+
assert.throws(
298+
() => readScreenshotScriptFlag({ args: ['--pixel-density', '0'], index: 0, flags: {} }),
299+
{ code: 'INVALID_ARGS', message: 'screenshot --pixel-density requires a positive integer' },
300+
);
301+
});

‎packages/command-surface/src/capture/screenshot.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog';
2+
import type { CommandSchemaOverride } from '@agent-device/command-registry/command-schema';
23
import type { CaptureScreenshotOptions } from '@agent-device/contracts/client';
34
import { SESSION_SURFACES } from '@agent-device/contracts/session';
45
import {
@@ -72,7 +73,17 @@ const screenshotCommandMetadata = defineFieldCommandMetadata(
7273
const screenshotCliSchema = {
7374
positionalArgs: ['path?'],
7475
allowedFlags: SCREENSHOT_COMMAND_FLAG_KEYS,
75-
} as const;
76+
// A burst arms a multi-file write, so only the invocation that names the flags may start one.
77+
// The count and wait serve press and swipe series too, and a repository `count` set for those
78+
// must not turn a single capture into a burst, nor refuse --crop-on beside it.
79+
explicitOnlyFlags: ['count', 'intervalMs'],
80+
flagDescriptionOverrides: {
81+
count:
82+
'Burst: capture this many frames of one animation in one request into a numbered file sequence (2-100). The CLI reads it only from this invocation; a config count for press and swipe series never arms a burst',
83+
intervalMs:
84+
'Burst: wait this many milliseconds between frames (100-10000, default 100). Requires --count, and like --count it is read only from this invocation',
85+
},
86+
} as const satisfies CommandSchemaOverride;
7687

7788
export const screenshotCliReader: CliReader = (positionals, flags) => ({
7889
...commonInputFromFlags(flags),
@@ -95,7 +106,7 @@ export const screenshotCommandFacet = defineCommandFacet({
95106
text: {
96107
summary: 'Capture a screenshot',
97108
cliDetail:
98-
"Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. Use --screen-mask black to fill the iOS simulator screen outline, including rounded corners, with black for this capture. It requires simctl: physical devices, tvOS and visionOS simulators, other platforms, provider captures and XCTest runner captures are refused. A masked capture fails if simctl fails instead of falling back to an unmasked runner capture. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on <selector> to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators, Android emulators, and CoreDevice physical iPhones), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Physical iPhone crop verification covers portrait CoreDevice/devicectl captures only. XCTest, runner fallback, and physical iPad crops are refused; landscape iPhone crop is unverified. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. Use --count with --interval-ms to capture an animation burst in one request. The daemon captures frames sequentially and writes each frame as a numbered file beside the --out path, for example page-0001.png, and the result reports every frame path and capture time. A burst refuses --crop-on and --overlay-refs, because a crop binds one snapshot and overlay refs annotate one frame. Keep --count times --interval-ms within 30 seconds, because the frames must finish inside the request timeout. When --count is present without --interval-ms, the daemon waits 100ms between frames. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent. On iOS simulators a screenshot waits for that simulator's status bar to be free, and a request its caller cancelled while it waits reports the cancellation instead of capturing.",
109+
"Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. Use --screen-mask black to fill the iOS simulator screen outline, including rounded corners, with black for this capture. It requires simctl: physical devices, tvOS and visionOS simulators, other platforms, provider captures and XCTest runner captures are refused. A masked capture fails if simctl fails instead of falling back to an unmasked runner capture. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on <selector> to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators, Android emulators, and CoreDevice physical iPhones), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Physical iPhone crop verification covers portrait CoreDevice/devicectl captures only. XCTest, runner fallback, and physical iPad crops are refused; landscape iPhone crop is unverified. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. Use --count with --interval-ms to capture an animation burst in one request. The daemon captures frames sequentially and writes each frame as a numbered file beside the --out path, for example page-0001.png, and the result reports every frame path and capture time. A burst refuses --crop-on and --overlay-refs, because a crop binds one snapshot and overlay refs annotate one frame. Keep --count times --interval-ms plus 500 ms per frame within 60 seconds, because the frames must finish inside the request timeout. When --count is present without --interval-ms, the daemon waits 100ms between frames. A burst starts only when this invocation names --count; a count set for press and swipe in a config file never arms one. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent. On iOS simulators a screenshot waits for that simulator's status bar to be free, and a request its caller cancelled while it waits reports the cancellation instead of capturing.",
99110
},
100111
metadata: screenshotCommandMetadata,
101112
run: (client, input) => client.capture.screenshot(input),

‎packages/contracts/src/screenshot.ts‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -102,18 +102,22 @@ export const SCREENSHOT_CAPTURE_BACKENDS = ['runner'] as const;
102102
export type ScreenshotCaptureBackend = (typeof SCREENSHOT_CAPTURE_BACKENDS)[number];
103103

104104
/**
105-
* The burst (`--count`) bounds a screenshot request admits. `maxCount` mirrors the shared
106-
* `--count` flag definition's ceiling, so CLI parse and this admission agree. `maxScheduledWaitMs`
107-
* bounds `count * intervalMs`, because frames are captured sequentially inside the request's
108-
* standard daemon budget; a longer run belongs in a batch of single captures.
105+
* The burst (`--count`) bounds a screenshot request admits. The shared `--count` flag definition
106+
* keeps the press/swipe ceiling, so this admission is the tighter screenshot bound. Frames run
107+
* sequentially inside the request's standard daemon budget, so the admission models the whole run,
108+
* not just the waits: `count * (intervalMs + perFrameBudgetMs)` stays within `maxRunMs`, which
109+
* leaves the request envelope margin for binding, transport, and a capture that exceeds its
110+
* allowance. A longer run belongs in a batch of single captures.
109111
*/
110112
export const SCREENSHOT_BURST_LIMITS = {
111113
minCount: 2,
112-
maxCount: 200,
114+
maxCount: 100,
113115
minIntervalMs: 100,
114116
maxIntervalMs: 10_000,
115117
defaultIntervalMs: 100,
116-
maxScheduledWaitMs: 30_000,
118+
/** The capture cost one frame budgets for; a sequential capture on a healthy target stays below it. */
119+
perFrameBudgetMs: 500,
120+
maxRunMs: 60_000,
117121
} as const;
118122

119123
export const SCREENSHOT_COMMAND_FLAG_KEYS = [
@@ -466,11 +470,14 @@ export function validateScreenshotBurst(
466470
`screenshot --interval-ms requires an integer from ${SCREENSHOT_BURST_LIMITS.minIntervalMs} to ${SCREENSHOT_BURST_LIMITS.maxIntervalMs}`,
467471
);
468472
}
469-
const scheduledWaitMs = count * (intervalMs ?? SCREENSHOT_BURST_LIMITS.defaultIntervalMs);
470-
if (scheduledWaitMs > SCREENSHOT_BURST_LIMITS.maxScheduledWaitMs) {
473+
const runMs =
474+
count *
475+
((intervalMs ?? SCREENSHOT_BURST_LIMITS.defaultIntervalMs) +
476+
SCREENSHOT_BURST_LIMITS.perFrameBudgetMs);
477+
if (runMs > SCREENSHOT_BURST_LIMITS.maxRunMs) {
471478
throw new AppError(
472479
'INVALID_ARGS',
473-
`screenshot --count schedules ${scheduledWaitMs} ms of waits; keep the burst within ${SCREENSHOT_BURST_LIMITS.maxScheduledWaitMs} ms by lowering --count or --interval-ms, or capture in a batch`,
480+
`screenshot --count schedules ${runMs} ms of waits and captures; keep the burst within ${SCREENSHOT_BURST_LIMITS.maxRunMs} ms by lowering --count or --interval-ms, or capture in a batch`,
474481
);
475482
}
476483
if (options.cropOn !== undefined || options.overlayRefs === true) {
@@ -583,7 +590,7 @@ function readScreenshotIntScriptFlag(params: {
583590
throw new AppError(
584591
'INVALID_ARGS',
585592
max === Number.MAX_SAFE_INTEGER
586-
? `${definition.label} requires an integer from ${min}`
593+
? `${definition.label} requires a positive integer`
587594
: `${definition.label} requires an integer from ${min} to ${max}`,
588595
);
589596
}

0 commit comments

Comments
 (0)