Skip to content

Commit 80cdf2c

Browse files
0xdaniellclaude
0xdaniell
and
claude
committed
feat(screenshot): report points for a native capture
A caller maps native pixels to tap coordinates with the display's scale, so return the measured density and logical size. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent d56632e commit 80cdf2c

7 files changed

Lines changed: 65 additions & 11 deletions

File tree

‎packages/capture-kit/src/screenshot-density.test.ts‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@ import assert from 'node:assert/strict';
22
import { test } from 'vitest';
33
import type { DeviceInfo } from '@agent-device/kernel/device';
44
import { AppError } from '@agent-device/kernel/errors';
5-
import { assertSupportedScreenshotPixelDensity } from './screenshot-density.ts';
5+
import { assertSupportedScreenshotPixelDensity, readScreenshotResultMetadata } from './screenshot-density.ts';
6+
import { mkdtemp, writeFile } from 'node:fs/promises';
7+
import os from 'node:os';
8+
import path from 'node:path';
69

710
type Device = Pick<DeviceInfo, 'platform' | 'appleOs' | 'kind'>;
811
const simulator: Device = { platform: 'apple', appleOs: 'ios', kind: 'simulator' };
@@ -25,3 +28,26 @@ test('native density is refused where density is unsupported', () => {
2528
(error: unknown) => error instanceof AppError && error.code === 'UNSUPPORTED_OPERATION',
2629
);
2730
});
31+
32+
test('a native capture reports its points, so a tap can be mapped from its pixels', async () => {
33+
const dir = await mkdtemp(path.join(os.tmpdir(), 'density-'));
34+
const file = path.join(dir, 'shot.png');
35+
const png = Buffer.alloc(33);
36+
Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).copy(png);
37+
png.writeUInt32BE(13, 8);
38+
png.write('IHDR', 12, 'latin1');
39+
png.writeUInt32BE(1206, 16);
40+
png.writeUInt32BE(2622, 20);
41+
await writeFile(file, png);
42+
assert.deepEqual(
43+
await readScreenshotResultMetadata({
44+
device: simulator,
45+
path: file,
46+
requestedPixelDensity: undefined,
47+
nativeDensity: true,
48+
sourcePixelDensity: 3,
49+
scale: undefined,
50+
}),
51+
{ width: 1206, height: 2622, logicalWidth: 402, logicalHeight: 874, pixelDensity: 3 },
52+
);
53+
});

‎packages/capture-kit/src/screenshot-density.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ export async function readScreenshotResultMetadata(params: {
4848
path: string;
4949
requestedPixelDensity: number | undefined;
5050
nativeDensity?: boolean;
51+
sourcePixelDensity?: number;
5152
scale: number | undefined;
5253
}): Promise<ScreenshotResultData> {
5354
const requiresDensityMetadata = supportsScreenshotPixelDensity(params.device);
@@ -64,13 +65,15 @@ export async function readScreenshotResultMetadata(params: {
6465
};
6566
if (
6667
!requiresDensityMetadata ||
67-
params.nativeDensity ||
6868
(params.scale !== undefined && params.scale !== 1)
6969
) {
7070
return result;
7171
}
7272

73-
const pixelDensity = resolveScreenshotPixelDensity(params.requestedPixelDensity);
73+
const pixelDensity = params.nativeDensity
74+
? params.sourcePixelDensity
75+
: resolveScreenshotPixelDensity(params.requestedPixelDensity);
76+
if (pixelDensity === undefined) return result;
7477
return {
7578
...result,
7679
logicalWidth: Math.round(size.width / pixelDensity),

‎packages/contracts/src/daemon-command-surface.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ export type DaemonScreenshotResult = Readonly<{
6767
path: string;
6868
artifacts?: ArtifactDescriptor[];
6969
displayRotation?: DeviceRotation;
70+
sourcePixelDensity?: number;
7071
message?: string;
7172
warnings?: string[];
7273
}>;

‎packages/contracts/src/interactor-types.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ export type ScreenshotOptions = {
7676
*/
7777
export type ScreenshotCaptureFacts = Readonly<{
7878
displayRotation?: DeviceRotation;
79+
/** Pixels per point of a capture kept at the display's own pixels. */
80+
sourcePixelDensity?: number;
7981
/** Actual physical iOS capture route, including any fallback from CoreDevice to XCTest. */
8082
iosPhysicalCaptureBackend?: 'coredevice' | 'xctest';
8183
}>;

‎packages/daemon/src/execution/screenshot-runtime.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@ export async function captureScreenshotArtifact(
158158
type CapturedScreenshot = Readonly<{
159159
path: string;
160160
displayRotation?: DeviceRotation;
161+
sourcePixelDensity?: number;
161162
message?: string;
162163
warnings?: string[];
163164
}>;
@@ -233,6 +234,7 @@ async function executeScreenshot(
233234
path: captured.path,
234235
requestedPixelDensity: flags?.screenshotPixelDensity,
235236
nativeDensity: flags?.screenshotNativeDensity,
237+
sourcePixelDensity: captured.sourcePixelDensity,
236238
scale: flags?.screenshotScale,
237239
})),
238240
};
@@ -398,7 +400,14 @@ function createBoundScreenshotBackend(
398400
});
399401
crop.run.warnings.push(...buildScreenshotCropWarnings(outcome));
400402
}
401-
return facts.displayRotation ? { displayRotation: facts.displayRotation } : undefined;
403+
return facts.displayRotation || facts.sourcePixelDensity !== undefined
404+
? {
405+
...(facts.displayRotation ? { displayRotation: facts.displayRotation } : {}),
406+
...(facts.sourcePixelDensity !== undefined
407+
? { pixelDensity: facts.sourcePixelDensity }
408+
: {}),
409+
}
410+
: undefined;
402411
},
403412
};
404413
}

‎packages/platform-apple/src/core/screenshot.ts‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -56,14 +56,15 @@ type SimulatorScreenshotFlowDeps = {
5656
* actually taken. `sourcePixelDensity` is that image's own measured scale — the panel the host
5757
* named for a `simctl` capture, or what the runner reported for a runner capture. Undefined means
5858
* nobody measured the source: a single-panel `simctl` capture, or a multi-panel device whose
59-
* display inventory never resolved, and the scale probe is the answer either way.
59+
* display inventory never resolved, and the scale probe is the answer either way. Resolves to
60+
* the source's scale when the file keeps its pixels (`native`), so a caller can map them to points.
6061
*/
6162
normalizeDensity: (
6263
device: DeviceInfo,
6364
outPath: string,
6465
pixelDensity: number | 'native' | undefined,
6566
sourcePixelDensity: number | undefined,
66-
) => Promise<void>;
67+
) => Promise<number | undefined>;
6768
captureWithRunner: (
6869
device: DeviceInfo,
6970
outPath: string,
@@ -149,10 +150,16 @@ export async function captureSimulatorScreenshotWithFallback(
149150
// A foldable lights one panel at a time, so the capture must name the panel the
150151
// system is currently showing rather than accept simctl's implicit default.
151152
const { display, displayRotation } = await deps.resolveCaptureTarget(device);
152-
const captureFacts: ScreenshotCaptureFacts = displayRotation ? { displayRotation } : {};
153+
const captureFacts: { -readonly [K in keyof ScreenshotCaptureFacts]: ScreenshotCaptureFacts[K] } =
154+
displayRotation ? { displayRotation } : {};
155+
const noteDensity = (sourcePixelDensity: number | undefined) => {
156+
if (sourcePixelDensity !== undefined) captureFacts.sourcePixelDensity = sourcePixelDensity;
157+
};
153158
const captureAndNormalize = async () => {
154159
await deps.captureWithRetry(device, outPath, display);
155-
await deps.normalizeDensity(device, outPath, options.pixelDensity, display?.pointScale);
160+
noteDensity(
161+
await deps.normalizeDensity(device, outPath, options.pixelDensity, display?.pointScale),
162+
);
156163
};
157164
let restoreStatusBar = async () => {};
158165
if (options.normalizeStatusBar === true) {
@@ -197,7 +204,9 @@ export async function captureSimulatorScreenshotWithFallback(
197204
// that image at, so normalization reads the source instead of applying a panel the host guessed.
198205
// A runner that reports nothing measured nothing: the probe stays the pre-panel answer rather
199206
// than borrowing a scale from a panel nobody captured (#2728).
200-
await deps.normalizeDensity(device, outPath, options.pixelDensity, captured?.pixelsPerPoint);
207+
noteDensity(
208+
await deps.normalizeDensity(device, outPath, options.pixelDensity, captured?.pixelsPerPoint),
209+
);
201210
return captureFacts;
202211
} finally {
203212
await restoreStatusBar().catch((error) =>
@@ -470,20 +479,21 @@ async function normalizeIosSimulatorScreenshotDensity(
470479
outPath: string,
471480
pixelDensity: number | 'native' | undefined,
472481
sourcePixelDensity?: number,
473-
): Promise<void> {
474-
if (pixelDensity === 'native') return;
482+
): Promise<number | undefined> {
475483
// Both the captured panel's `pointScale` and the runner's reported scale describe the image that
476484
// was actually taken. `SIMULATOR_MAINSCREEN_SCALE` describes one fixed panel, so on a foldable it
477485
// can disagree with the panel that was captured, and it is only ever consulted when nothing
478486
// measured the source — which a multi-panel capture and every runner capture do.
479487
const measuredSourcePixelDensity =
480488
sourcePixelDensity ?? (await readIosSimulatorMainScreenScale(device));
489+
if (pixelDensity === 'native') return measuredSourcePixelDensity;
481490
const targetSize = computeDensityScaledScreenshotSize(
482491
await readPngSize(outPath),
483492
measuredSourcePixelDensity,
484493
pixelDensity,
485494
);
486495
if (targetSize) await resizePngFile(outPath, targetSize.width, targetSize.height);
496+
return undefined;
487497
}
488498

489499
async function readIosSimulatorMainScreenScale(device: DeviceInfo): Promise<number> {

‎src/commands/capture/runtime/screenshot.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ export const screenshotCommand: RuntimeCommand<
3131

3232
let artifact: ArtifactDescriptor | undefined;
3333
let displayRotation: DeviceRotation | undefined;
34+
let sourcePixelDensity: number | undefined;
3435
try {
3536
const captured = await runtime.backend.captureScreenshot(
3637
{
@@ -52,6 +53,7 @@ export const screenshotCommand: RuntimeCommand<
5253
},
5354
);
5455
displayRotation = captured?.displayRotation;
56+
sourcePixelDensity = captured?.pixelDensity;
5557
if (options.scale !== undefined) {
5658
await resizePngFileToScale(reserved.path, options.scale);
5759
}
@@ -65,6 +67,7 @@ export const screenshotCommand: RuntimeCommand<
6567
path: reserved.path,
6668
...(artifact ? { artifacts: [artifact] } : {}),
6769
...(displayRotation ? { displayRotation } : {}),
70+
...(sourcePixelDensity !== undefined ? { sourcePixelDensity } : {}),
6871
...successText(`Saved screenshot: ${reserved.path}`),
6972
};
7073
};

0 commit comments

Comments
 (0)