Skip to content

Commit 0cd679f

Browse files
fix(limrun): write the iOS screenshot as PNG instead of Limrun's JPEG bytes (#2645)
* fix(limrun): write the iOS screenshot as PNG instead of Limrun's JPEG bytes Limrun serves its simulator capture as JPEG and the interactor wrote those bytes straight to the .png path, so every screenshot on a Limrun iOS direct session failed downstream with "Screenshot file is not a valid PNG". Sniff the container and transcode a JPEG to PNG before writing; a PNG passes through unchanged. The transcode runs as a PNG worker job with the usual in-process fallback so a full-resolution capture never blocks the daemon event loop, and a JPEG whose body does not decode fails with the labeled COMMAND_FAILED the PNG decoder uses. jpeg-js is bundled like pngjs and loaded only on the screenshot path. * refactor(capture-kit): keep the JPEG transcode behind the PNG worker The Limrun screenshot path reaches the transcoder only through png-worker-client, so the ./png-transcode subpath and its layering-guard entry are dropped. The PNG signature check lives in png.ts, which both the worker client and the transcoder already load, so the two sniffs cannot drift and no entry's eager closure grows. The corrupt-JPEG worker test now pins details.label and details.reason across the worker boundary. Also restores the root package.json dependency order main has. * perf(capture-kit): load the JPEG decoder in the PNG worker only for a transcode job The daemon prewarms the PNG worker at startup, and its static import of png-transcode evaluated jpeg-js in every daemon's worker thread whether or not a Limrun screenshot was ever taken. The jpeg-to-png case now imports the transcoder on first use. --------- Co-authored-by: Michał Pierzchała <[email protected]>
1 parent f4930d6 commit 0cd679f

16 files changed

Lines changed: 316 additions & 16 deletions

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22

33
## Unreleased
44

5+
- Fixed (limrun): `screenshot` on Limrun iOS direct sessions writes a PNG. Limrun serves its capture
6+
as JPEG and the interactor wrote those bytes straight to the `.png` path, so every capture failed
7+
downstream with "Screenshot file is not a valid PNG". The bytes are now sniffed and a JPEG is
8+
transcoded to PNG before it is written; a PNG passes through unchanged.
59
- Changed (ios): a regular `snapshot --depth N` on the XCTest runner is a presentation cut over a
610
full acquisition, not a bound on the walk. Acquisition publishes the frames the platform reported,
711
one normalization pass turns geometry into the app's orientation space and recomputes `hittable`,

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,7 @@
318318
"fallow": "^2.104.0",
319319
"fast-check": "^4.9.0",
320320
"ipaddr.js": "^2.5.0",
321+
"jpeg-js": "^0.4.4",
321322
"oxc-parser": "^0.146.0",
322323
"oxfmt": "^0.64.0",
323324
"oxlint": "^1.79.0",

‎packages/capture-kit/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,7 @@
290290
"devDependencies": {
291291
"@types/pngjs": "^6.0.5",
292292
"fast-check": "^4.9.0",
293+
"jpeg-js": "^0.4.4",
293294
"pngjs": "^7.0.0"
294295
}
295296
}
Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
import { expect, test } from 'vitest';
2+
import { encode as encodeJpeg } from 'jpeg-js';
3+
import { PNG } from './png.ts';
4+
import { detectScreenshotImageFormat, transcodeScreenshotToPng } from './png-transcode.ts';
5+
6+
function solidRgba(width: number, height: number, rgba: readonly [number, number, number, number]) {
7+
const data = Buffer.alloc(width * height * 4);
8+
for (let offset = 0; offset < data.length; offset += 4) data.set(rgba, offset);
9+
return data;
10+
}
11+
12+
/** The value a synchronous call throws, so one assertion can check its code and message together. */
13+
function thrownBy(run: () => unknown): unknown {
14+
try {
15+
run();
16+
} catch (error) {
17+
return error;
18+
}
19+
throw new Error('expected the call to throw');
20+
}
21+
22+
test('a JPEG screenshot becomes a PNG of the same size with the same picture', () => {
23+
const jpeg = encodeJpeg({ width: 6, height: 4, data: solidRgba(6, 4, [200, 30, 30, 255]) }, 100);
24+
expect(detectScreenshotImageFormat(jpeg.data)).toBe('jpeg');
25+
26+
const png = transcodeScreenshotToPng(jpeg.data, 'test screenshot');
27+
28+
expect(detectScreenshotImageFormat(png)).toBe('png');
29+
const decoded = PNG.sync.read(png);
30+
expect([decoded.width, decoded.height]).toEqual([6, 4]);
31+
const [r = -1, g = -1, b = -1, a = -1] = decoded.data.subarray(0, 4);
32+
// JPEG is lossy; a flat field survives within a few levels per channel.
33+
expect(Math.abs(r - 200)).toBeLessThanOrEqual(4);
34+
expect(Math.abs(g - 30)).toBeLessThanOrEqual(4);
35+
expect(Math.abs(b - 30)).toBeLessThanOrEqual(4);
36+
expect(a).toBe(255);
37+
});
38+
39+
test('a PNG screenshot passes through byte for byte', () => {
40+
const png = PNG.sync.write(new PNG({ width: 3, height: 2 }));
41+
42+
expect(transcodeScreenshotToPng(png, 'test screenshot')).toBe(png);
43+
});
44+
45+
test('bytes that are neither container are refused with a typed error carrying the label', () => {
46+
expect(
47+
thrownBy(() => transcodeScreenshotToPng(Buffer.from('GIF89a'), 'Limrun iOS screenshot')),
48+
).toMatchObject({
49+
code: 'COMMAND_FAILED',
50+
message: 'Limrun iOS screenshot is neither PNG nor JPEG',
51+
details: { label: 'Limrun iOS screenshot', leadingBytes: '47494638' },
52+
});
53+
});
54+
55+
test('a valid SOI marker followed by garbage is refused with the typed decode error, not the decoder exception', () => {
56+
const garbage = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(32, 0x41)]);
57+
expect(detectScreenshotImageFormat(garbage)).toBe('jpeg');
58+
59+
expect(thrownBy(() => transcodeScreenshotToPng(garbage, 'Limrun iOS screenshot'))).toMatchObject({
60+
code: 'COMMAND_FAILED',
61+
message: 'Failed to decode Limrun iOS screenshot as JPEG',
62+
details: { label: 'Limrun iOS screenshot', reason: expect.any(String) },
63+
});
64+
});
65+
66+
test('a truncated JPEG body is refused with the same typed decode error', () => {
67+
const jpeg = encodeJpeg({ width: 6, height: 4, data: solidRgba(6, 4, [10, 200, 30, 255]) }, 100);
68+
const truncated = jpeg.data.subarray(0, 24);
69+
70+
expect(
71+
thrownBy(() => transcodeScreenshotToPng(truncated, 'Limrun iOS screenshot')),
72+
).toMatchObject({
73+
code: 'COMMAND_FAILED',
74+
message: 'Failed to decode Limrun iOS screenshot as JPEG',
75+
});
76+
});
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import { AppError } from '@agent-device/kernel/errors';
2+
import { decode as decodeJpeg } from 'jpeg-js';
3+
import { hasPngSignature, PNG } from './png.ts';
4+
5+
const JPEG_SIGNATURE = Buffer.from([0xff, 0xd8, 0xff]);
6+
7+
export type ScreenshotImageFormat = 'png' | 'jpeg';
8+
9+
/** Names the container a screenshot arrived in from its magic bytes, or `undefined` for neither. */
10+
export function detectScreenshotImageFormat(bytes: Buffer): ScreenshotImageFormat | undefined {
11+
if (hasPngSignature(bytes)) return 'png';
12+
if (bytes.subarray(0, JPEG_SIGNATURE.length).equals(JPEG_SIGNATURE)) return 'jpeg';
13+
return undefined;
14+
}
15+
16+
/**
17+
* Returns PNG bytes for a screenshot a provider handed back in whatever container it prefers. PNG
18+
* passes through untouched; JPEG is decoded and re-encoded losslessly from the decoded pixels, so
19+
* every PNG-only reader downstream (size, crop, overlay, diff) sees the format the path promises.
20+
*
21+
* Synchronous and CPU-bound: daemon request paths reach it through the PNG worker
22+
* (`transcodeScreenshotToPngAsync` in `png-worker-client.ts`), like every other codec job.
23+
*/
24+
export function transcodeScreenshotToPng(bytes: Buffer, label: string): Buffer {
25+
const format = detectScreenshotImageFormat(bytes);
26+
if (format === 'png') return bytes;
27+
if (format === 'jpeg') return jpegToPng(bytes, label);
28+
throw new AppError('COMMAND_FAILED', `${label} is neither PNG nor JPEG`, {
29+
label,
30+
leadingBytes: bytes.subarray(0, 4).toString('hex'),
31+
});
32+
}
33+
34+
/** A JPEG signature does not prove a decodable body; a failure keeps the label and the decoder's reason. */
35+
function jpegToPng(bytes: Buffer, label: string): Buffer {
36+
try {
37+
const decoded = decodeJpeg(bytes, { useTArray: true, formatAsRGBA: true });
38+
const png = new PNG({ width: decoded.width, height: decoded.height });
39+
png.data = Buffer.from(decoded.data.buffer, decoded.data.byteOffset, decoded.data.byteLength);
40+
return PNG.sync.write(png);
41+
} catch (error) {
42+
throw new AppError('COMMAND_FAILED', `Failed to decode ${label} as JPEG`, {
43+
label,
44+
reason: error instanceof Error ? error.message : String(error),
45+
});
46+
}
47+
}

‎packages/capture-kit/src/png-worker-client.test.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
decodePngAsync,
99
encodePngAsync,
1010
terminatePngWorker,
11+
transcodeScreenshotToPngAsync,
1112
} from './png-worker-client.ts';
1213
import { computeScreenshotDiffPixels } from './screenshot-diff-pixels.ts';
1314

@@ -106,3 +107,34 @@ test('decodePngAsync rejects invalid PNG data with the canonical decode AppError
106107
},
107108
);
108109
});
110+
111+
test('transcodeScreenshotToPngAsync matches the synchronous transcoder byte for byte', async () => {
112+
const { encode } = await import('jpeg-js');
113+
const { transcodeScreenshotToPng } = await import('./png-transcode.ts');
114+
const rgba = Buffer.alloc(9 * 7 * 4, 0x66);
115+
const jpeg = encode({ width: 9, height: 7, data: rgba }, 90).data;
116+
117+
const fromWorker = await transcodeScreenshotToPngAsync(jpeg, 'fixture');
118+
119+
assert.deepEqual(fromWorker, transcodeScreenshotToPng(jpeg, 'fixture'));
120+
assert.equal(PNG.sync.read(fromWorker).width, 9);
121+
});
122+
123+
test('transcodeScreenshotToPngAsync returns a PNG untouched without a worker round trip', async () => {
124+
const png = PNG.sync.write(new PNG({ width: 2, height: 2 }));
125+
126+
assert.equal(await transcodeScreenshotToPngAsync(png, 'fixture'), png);
127+
});
128+
129+
test('transcodeScreenshotToPngAsync rejects a corrupt JPEG with the canonical decode AppError', async () => {
130+
const corrupt = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(16, 0)]);
131+
132+
await assert.rejects(transcodeScreenshotToPngAsync(corrupt, 'fixture'), (error) => {
133+
assert.equal(error instanceof AppError, true);
134+
assert.equal((error as AppError).code, 'COMMAND_FAILED');
135+
assert.match((error as AppError).message, /Failed to decode fixture as JPEG/);
136+
assert.equal((error as AppError).details?.label, 'fixture');
137+
assert.ok(String((error as AppError).details?.reason).length > 0);
138+
return true;
139+
});
140+
});

‎packages/capture-kit/src/png-worker-client.ts‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { emitDiagnostic } from '@agent-device/host-kit/diagnostics';
33
import { AppError, toAppErrorCode } from '@agent-device/kernel/errors';
44
import type { Rect } from '@agent-device/kernel/snapshot';
55
import { resolveInternalEntryModulePath } from './internal-entry.ts';
6-
import { decodePng, PNG } from './png.ts';
6+
import { decodePng, hasPngSignature, PNG } from './png.ts';
77
import {
88
computeScreenshotDiffPixels,
99
type ScreenshotDiffPixelsJob,
@@ -256,3 +256,19 @@ export async function computeScreenshotDiffPixelsAsync(
256256
}));
257257
return { ...result, diffData: toBuffer(result.diffData) };
258258
}
259+
260+
/**
261+
* PNG bytes for a provider screenshot in whatever container it arrived in. A PNG returns as is
262+
* without a worker round trip; a JPEG is decoded and re-encoded on the worker so a full-resolution
263+
* capture never blocks the daemon event loop. Decode failures carry the canonical `AppError`.
264+
*/
265+
export async function transcodeScreenshotToPngAsync(bytes: Buffer, label: string): Promise<Buffer> {
266+
if (hasPngSignature(bytes)) return bytes;
267+
const result = await runPngJob({ kind: 'jpeg-to-png', image: bytes, label }, async () => {
268+
// Read on demand so the JPEG decoder stays out of the import closure of every entry that only
269+
// needs the worker's other jobs.
270+
const { transcodeScreenshotToPng } = await import('./png-transcode.ts');
271+
return { kind: 'jpeg-to-png', png: transcodeScreenshotToPng(bytes, label) };
272+
});
273+
return toBuffer(result.png);
274+
}

‎packages/capture-kit/src/png-worker-contract.ts‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import type { PngRgbDifferenceResult } from './png-rgb-difference.ts';
99
/**
1010
* Message contract between the daemon-side PNG worker client
1111
* (`png-worker-client.ts`) and the worker thread entry (`png-worker.ts`).
12-
* One message = one decode, encode, crop, or diff job. Binary payloads cross the
12+
* One message = one decode, encode, crop, diff, or transcode job. Binary payloads cross the
1313
* thread boundary via structured clone (or transfer), so `Buffer` fields
1414
* arrive as plain `Uint8Array` views on the receiving side.
1515
*/
@@ -19,15 +19,18 @@ export type PngWorkerJob =
1919
| { kind: 'encode'; width: number; height: number; data: Uint8Array }
2020
| { kind: 'crop'; png: Uint8Array; label: string; box: Rect }
2121
| { kind: 'rgb-difference'; firstPng: Uint8Array; secondPng: Uint8Array; label: string }
22-
| ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsJob);
22+
| ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsJob)
23+
// A provider screenshot in whatever container it arrived in; answers PNG bytes.
24+
| { kind: 'jpeg-to-png'; image: Uint8Array; label: string };
2325

2426
export type PngWorkerJobResult =
2527
| { kind: 'decode'; width: number; height: number; data: Uint8Array }
2628
| { kind: 'encode'; png: Uint8Array }
2729
// A crop answers `null` when the box already covers the image, so the caller keeps the file.
2830
| { kind: 'crop'; png: Uint8Array | null }
2931
| ({ kind: 'rgb-difference' } & PngRgbDifferenceResult)
30-
| ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsResult);
32+
| ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsResult)
33+
| { kind: 'jpeg-to-png'; png: Uint8Array };
3134

3235
export type PngWorkerJobKind = PngWorkerJob['kind'];
3336

‎packages/capture-kit/src/png-worker.ts‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@ import {
1717
* `png-worker-client.ts`; published as the `internal/png-worker` build entry.
1818
*/
1919

20-
function runJob(request: PngWorkerRequest): PngWorkerJobResult {
20+
// The daemon prewarms this worker at startup, so the JPEG decoder loads only for a transcode job.
21+
async function runJob(request: PngWorkerRequest): Promise<PngWorkerJobResult> {
2122
switch (request.kind) {
2223
case 'decode': {
2324
const png = decodePng(toBuffer(request.png), request.label);
@@ -42,6 +43,13 @@ function runJob(request: PngWorkerRequest): PngWorkerJobResult {
4243
case 'diff-pixels': {
4344
return { kind: 'diff-pixels', ...computeScreenshotDiffPixels(request) };
4445
}
46+
case 'jpeg-to-png': {
47+
const { transcodeScreenshotToPng } = await import('./png-transcode.ts');
48+
return {
49+
kind: 'jpeg-to-png',
50+
png: transcodeScreenshotToPng(toBuffer(request.image), request.label),
51+
};
52+
}
4553
}
4654
}
4755

@@ -81,15 +89,17 @@ function resultBufferViews(result: PngWorkerJobResult): Uint8Array[] {
8189
return [];
8290
case 'diff-pixels':
8391
return [result.diffData, result.diffMask];
92+
case 'jpeg-to-png':
93+
return [result.png];
8494
}
8595
}
8696

8797
const port = parentPort;
8898
if (port) {
89-
port.on('message', (request: PngWorkerRequest) => {
99+
port.on('message', async (request: PngWorkerRequest) => {
90100
let response: PngWorkerResponse;
91101
try {
92-
response = { id: request.id, ok: true, result: runJob(request) };
102+
response = { id: request.id, ok: true, result: await runJob(request) };
93103
} catch (error) {
94104
response = { id: request.id, ok: false, error: normalizeError(error) };
95105
}

‎packages/capture-kit/src/png.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,13 @@ import { PNG } from 'pngjs';
33

44
export { PNG };
55

6+
const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
7+
8+
/** Whether the bytes open with the PNG signature, before any decode is attempted. */
9+
export function hasPngSignature(bytes: Buffer): boolean {
10+
return bytes.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE);
11+
}
12+
613
/**
714
* Decodes a PNG, wrapping failures in the canonical decode `AppError`. Shared
815
* by the in-process sync path and the PNG worker thread (`png-worker.ts`), so

0 commit comments

Comments
 (0)