Skip to content

Commit 3ed12d5

Browse files
committed
perf(screenshot): read the crop region instead of decoding the whole capture
`screenshot --crop-on` paid for a full PNG decode and an RGBA re-encode of the capture before keeping a frame. One worker job now turns the captured bytes into the cropped bytes: a region reader that stops at the box's last row, and a truecolor writer that drops the alpha channel when the cropped pixels carry none. The reader claims the 8-bit non-interlaced truecolor layout that iOS simulator and Android emulator captures arrive in. Palette, grayscale, interlaced, 16-bit, and structurally untrusted files fall through to the general PNG reader, which keeps owning the canonical decode error and the previous RGBA output. Measured on real captures, the crop stage is 1.9x to 5.1x faster and the artifact is 1.07x to 2.4x smaller. Cropped bytes verified pixel-for-pixel against ImageMagick's own crop across RGB, RGBA, grayscale, palette, 16-bit, interlaced, and translucent sources, on both iOS simulator and Android emulator captures.
1 parent 93a9b14 commit 3ed12d5

18 files changed

Lines changed: 1410 additions & 27 deletions
Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
import zlib from 'node:zlib';
2+
import { predictByte } from './png-predictor.ts';
3+
import { PNG } from './png.ts';
4+
5+
/**
6+
* Test-only PNG writer. `pngjs` picks one filter for a whole image and cannot express the
7+
* per-row mixtures, palettes, transparency, and header variations the crop paths have to
8+
* classify, so fixtures assemble the chunks directly. Every fixture is read back by `pngjs`
9+
* in the test that uses it, which is what proves this writer is honest.
10+
*/
11+
12+
const SIGNATURE = Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
13+
const FILTER_CYCLE: readonly number[] = [0, 1, 2, 3, 4];
14+
const IHDR_COLOR_TYPE_BYTE = 25;
15+
const IHDR_BIT_DEPTH_BYTE = 24;
16+
17+
export type PngFixture = Readonly<{
18+
pixels: Uint8Array;
19+
width: number;
20+
height: number;
21+
channels: number;
22+
colorType: number;
23+
bitDepth?: number;
24+
interlace?: number;
25+
compressionMethod?: number;
26+
filterMethod?: number;
27+
filterFor?: (row: number) => number;
28+
palette?: Uint8Array;
29+
transparency?: Uint8Array;
30+
ancillary?: Readonly<{ type: string; data: Uint8Array }>;
31+
}>;
32+
33+
export function encodeFixturePng(fixture: PngFixture): Buffer {
34+
const { height, width } = fixture;
35+
const header = Buffer.alloc(13);
36+
header.writeUInt32BE(width, 0);
37+
header.writeUInt32BE(height, 4);
38+
header[8] = fixture.bitDepth ?? 8;
39+
header[9] = fixture.colorType;
40+
header[10] = fixture.compressionMethod ?? 0;
41+
header[11] = fixture.filterMethod ?? 0;
42+
header[12] = fixture.interlace ?? 0;
43+
const chunks: Uint8Array[] = [SIGNATURE, pngChunk('IHDR', header)];
44+
if (fixture.ancillary) chunks.push(pngChunk(fixture.ancillary.type, fixture.ancillary.data));
45+
if (fixture.palette) chunks.push(pngChunk('PLTE', fixture.palette));
46+
if (fixture.transparency) chunks.push(pngChunk('tRNS', fixture.transparency));
47+
chunks.push(pngChunk('IDAT', zlib.deflateSync(filterScanlines(fixture), { level: 6 })));
48+
chunks.push(pngChunk('IEND', new Uint8Array(0)));
49+
return Buffer.concat(chunks);
50+
}
51+
52+
/** A pixel grid where neighbouring pixels differ, so a wrong row or column cannot hide. */
53+
export function rampPixels(
54+
width: number,
55+
height: number,
56+
channels: number,
57+
alpha: (x: number, y: number) => number = () => 255,
58+
): Uint8Array {
59+
const pixels = new Uint8Array(width * height * channels);
60+
for (let y = 0; y < height; y += 1) {
61+
for (let x = 0; x < width; x += 1) {
62+
const offset = (y * width + x) * channels;
63+
pixels[offset] = (x * 7 + y * 3) & 0xff;
64+
if (channels >= 3) {
65+
pixels[offset + 1] = (x * 11 + y * 5) & 0xff;
66+
pixels[offset + 2] = (x * 13 + y * 17) & 0xff;
67+
}
68+
if (channels === 2) pixels[offset + 1] = alpha(x, y) & 0xff;
69+
if (channels === 4) pixels[offset + 3] = alpha(x, y) & 0xff;
70+
}
71+
}
72+
return pixels;
73+
}
74+
75+
/** Reads any encoded PNG back as RGBA through `pngjs`, with the layout bytes it declares. */
76+
export function readPngForTest(buffer: Buffer): {
77+
width: number;
78+
height: number;
79+
colorType: number;
80+
bitDepth: number;
81+
rgba: Uint8Array;
82+
} {
83+
const png = PNG.sync.read(buffer);
84+
return {
85+
width: png.width,
86+
height: png.height,
87+
colorType: buffer.readUInt8(IHDR_COLOR_TYPE_BYTE),
88+
bitDepth: buffer.readUInt8(IHDR_BIT_DEPTH_BYTE),
89+
rgba: png.data,
90+
};
91+
}
92+
93+
function filterScanlines(fixture: PngFixture): Uint8Array {
94+
const { channels, height, width } = fixture;
95+
const stride = width * channels;
96+
const scanlines = new Uint8Array(height * (stride + 1));
97+
for (let row = 0; row < height; row += 1) {
98+
const data = row * (stride + 1) + 1;
99+
const filter = fixture.filterFor?.(row) ?? FILTER_CYCLE[row % FILTER_CYCLE.length]!;
100+
scanlines[row * (stride + 1)] = filter;
101+
const from = row * stride;
102+
for (let offset = 0; offset < stride; offset += 1) {
103+
scanlines[data + offset] =
104+
(fixture.pixels[from + offset]! -
105+
predictByte(
106+
filter,
107+
offset >= channels ? fixture.pixels[from + offset - channels]! : 0,
108+
row > 0 ? fixture.pixels[from - stride + offset]! : 0,
109+
row > 0 && offset >= channels ? fixture.pixels[from - stride + offset - channels]! : 0,
110+
)) &
111+
0xff;
112+
}
113+
}
114+
return scanlines;
115+
}
116+
117+
function pngChunk(type: string, data: Uint8Array): Buffer {
118+
const chunk = Buffer.alloc(12 + data.length);
119+
chunk.writeUInt32BE(data.length, 0);
120+
chunk.write(type, 4, 'ascii');
121+
chunk.set(data, 8);
122+
chunk.writeUInt32BE(zlib.crc32(chunk.subarray(4, 8 + data.length)) >>> 0, 8 + data.length);
123+
return chunk;
124+
}
125+
126+
/** Flips the bits of the byte `fromEnd` positions before the end, corrupting a checksum on purpose. */
127+
export function corruptFromEnd(buffer: Buffer, fromEnd: number): Buffer {
128+
const position = buffer.length - fromEnd;
129+
buffer[position] = (buffer[position] ?? 0) ^ 0xff;
130+
return buffer;
131+
}
Lines changed: 262 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,262 @@
1+
import { test } from 'vitest';
2+
import assert from 'node:assert/strict';
3+
import fc from 'fast-check';
4+
import type { Rect } from '@agent-device/kernel/snapshot';
5+
import { cropPngBytes } from './png-crop-bytes.ts';
6+
import {
7+
corruptFromEnd,
8+
encodeFixturePng,
9+
rampPixels,
10+
readPngForTest,
11+
type PngFixture,
12+
} from './png-codec.fixtures.ts';
13+
14+
const OPAQUE = 255;
15+
const LABEL = 'screenshot';
16+
const PROPERTY_RUNS = 100;
17+
18+
test('a mixed-filter RGBA capture crops to the box pixels and drops its alpha channel', () => {
19+
const image = claimable(9, 6);
20+
const box: Rect = { x: 2, y: 1, width: 4, height: 3 };
21+
22+
const cropped = cropPngBytes(encodeFixturePng(image), box, LABEL)!;
23+
24+
assert.equal(readPngForTest(cropped).colorType, 2, 'an opaque crop keeps no alpha channel');
25+
assertCroppedPixels(cropped, image, box);
26+
});
27+
28+
test('an RGB capture crops without gaining an alpha channel', () => {
29+
const image: PngFixture = {
30+
pixels: rampPixels(7, 5, 3),
31+
width: 7,
32+
height: 5,
33+
channels: 3,
34+
colorType: 2,
35+
};
36+
const box: Rect = { x: 1, y: 2, width: 3, height: 2 };
37+
38+
const cropped = cropPngBytes(encodeFixturePng(image), box, LABEL)!;
39+
40+
assert.equal(readPngForTest(cropped).colorType, 2);
41+
assertCroppedPixels(cropped, image, box);
42+
});
43+
44+
test('a one-pixel crop of the last row and column reads the right pixel', () => {
45+
const image = claimable(6, 4);
46+
const box: Rect = { x: 5, y: 3, width: 1, height: 1 };
47+
48+
assertCroppedPixels(cropPngBytes(encodeFixturePng(image), box, LABEL)!, image, box);
49+
});
50+
51+
test('a box holding a translucent pixel keeps the alpha channel', () => {
52+
const image = translucentAt(8, 5, { x: 3, y: 2 });
53+
const box: Rect = { x: 2, y: 1, width: 4, height: 3 };
54+
55+
const cropped = cropPngBytes(encodeFixturePng(image), box, LABEL)!;
56+
57+
const decoded = readPngForTest(cropped);
58+
assert.equal(decoded.colorType, 6);
59+
assert.equal(decoded.rgba[(1 * box.width + 1) * 4 + 3], 128);
60+
assertCroppedPixels(cropped, image, box);
61+
});
62+
63+
test('translucency outside the box still lets the crop drop alpha', () => {
64+
const image = translucentAt(8, 5, { x: 7, y: 4 });
65+
const box: Rect = { x: 0, y: 0, width: 4, height: 4 };
66+
67+
const cropped = cropPngBytes(encodeFixturePng(image), box, LABEL)!;
68+
69+
assert.equal(readPngForTest(cropped).colorType, 2);
70+
assertCroppedPixels(cropped, image, box);
71+
});
72+
73+
test('a palette capture crops through the general reader with its colors intact', () => {
74+
const width = 6;
75+
const height = 5;
76+
const palette = new Uint8Array(3 * 4);
77+
for (let color = 0; color < 4; color += 1) {
78+
palette[color * 3] = (color * 60) & 0xff;
79+
palette[color * 3 + 1] = (color * 90) & 0xff;
80+
palette[color * 3 + 2] = (color * 120) & 0xff;
81+
}
82+
const indices = new Uint8Array(width * height);
83+
for (let pixel = 0; pixel < indices.length; pixel += 1) indices[pixel] = pixel % 4;
84+
const box: Rect = { x: 1, y: 1, width: 3, height: 2 };
85+
86+
const cropped = cropPngBytes(
87+
encodeFixturePng({
88+
pixels: indices,
89+
width,
90+
height,
91+
channels: 1,
92+
colorType: 3,
93+
palette,
94+
filterFor: () => 0,
95+
}),
96+
box,
97+
LABEL,
98+
)!;
99+
100+
const decoded = readPngForTest(cropped);
101+
assert.equal(decoded.colorType, 6, 'the general reader keeps its RGBA output');
102+
for (let row = 0; row < box.height; row += 1) {
103+
for (let column = 0; column < box.width; column += 1) {
104+
const color = indices[(row + box.y) * width + column + box.x]!;
105+
const to = (row * box.width + column) * 4;
106+
assert.deepEqual(
107+
[decoded.rgba[to], decoded.rgba[to + 1], decoded.rgba[to + 2]],
108+
[palette[color * 3], palette[color * 3 + 1], palette[color * 3 + 2]],
109+
`pixel ${column},${row}`,
110+
);
111+
}
112+
}
113+
});
114+
115+
test('a grayscale capture crops through the general reader', () => {
116+
const width = 5;
117+
const height = 4;
118+
const gray = new Uint8Array(width * height);
119+
for (let pixel = 0; pixel < gray.length; pixel += 1) gray[pixel] = (pixel * 21) & 0xff;
120+
const box: Rect = { x: 2, y: 1, width: 2, height: 2 };
121+
122+
const cropped = cropPngBytes(
123+
encodeFixturePng({ pixels: gray, width, height, channels: 1, colorType: 0 }),
124+
box,
125+
LABEL,
126+
)!;
127+
128+
const decoded = readPngForTest(cropped);
129+
for (let row = 0; row < box.height; row += 1) {
130+
for (let column = 0; column < box.width; column += 1) {
131+
const wanted = gray[(row + box.y) * width + column + box.x]!;
132+
const to = (row * box.width + column) * 4;
133+
assert.deepEqual(
134+
[decoded.rgba[to], decoded.rgba[to + 1], decoded.rgba[to + 2]],
135+
[wanted, wanted, wanted],
136+
`pixel ${column},${row}`,
137+
);
138+
}
139+
}
140+
});
141+
142+
test('a box covering the whole image returns no replacement bytes', () => {
143+
assert.equal(cropPngBytes(encodeFixturePng(claimable(6, 4)), fullBox(6, 4), LABEL), null);
144+
});
145+
146+
test('a box beyond the image reports the box instead of clamping the crop', () => {
147+
assert.throws(
148+
() =>
149+
cropPngBytes(encodeFixturePng(claimable(6, 4)), { x: 4, y: 0, width: 3, height: 2 }, LABEL),
150+
/Screenshot crop box 3x2 at \(4, 0\) exceeds the 6x4 image/,
151+
);
152+
});
153+
154+
test('bytes that are not a PNG report the canonical decode failure', () => {
155+
assert.throws(
156+
() => cropPngBytes(Buffer.from('not a png at all'), { x: 0, y: 0, width: 2, height: 2 }, LABEL),
157+
/Failed to decode screenshot as PNG/,
158+
);
159+
});
160+
161+
test('a file truncated inside its image data reports the decode failure', () => {
162+
const buffer = encodeFixturePng(claimable(12, 9));
163+
164+
assert.throws(
165+
() =>
166+
cropPngBytes(
167+
Buffer.from(buffer.subarray(0, buffer.length - 8)),
168+
{ ...fullBox(12, 9), width: 4, height: 4 },
169+
LABEL,
170+
),
171+
/Failed to decode screenshot as PNG/,
172+
);
173+
});
174+
175+
test('a corrupted image-data checksum is refused instead of cropping wrong pixels', () => {
176+
const corrupted = corruptFromEnd(Buffer.from(encodeFixturePng(claimable(12, 9))), 20);
177+
178+
assert.throws(
179+
() => cropPngBytes(corrupted, { x: 0, y: 0, width: 4, height: 4 }, LABEL),
180+
/Failed to decode screenshot as PNG/,
181+
);
182+
});
183+
184+
test('every crop of a random capture matches the pixels the file declares', () => {
185+
fc.assert(
186+
fc.property(
187+
fc.integer({ min: 1, max: 24 }),
188+
fc.integer({ min: 1, max: 24 }),
189+
fc.nat(),
190+
fc.array(fc.integer({ min: 0, max: 4 }), { minLength: 1 }),
191+
(width, height, boxSeed, filterPlan) => {
192+
const image: PngFixture = {
193+
...claimable(width, height),
194+
filterFor: (row) => filterPlan[row % filterPlan.length]!,
195+
};
196+
const x = boxSeed % width;
197+
const y = (boxSeed >>> 4) % height;
198+
const box: Rect = {
199+
x,
200+
y,
201+
width: 1 + (boxSeed % (width - x)),
202+
height: 1 + ((boxSeed >>> 8) % (height - y)),
203+
};
204+
205+
const cropped = cropPngBytes(encodeFixturePng(image), box, LABEL);
206+
207+
if (box.x === 0 && box.y === 0 && box.width === width && box.height === height) {
208+
assert.equal(cropped, null);
209+
return;
210+
}
211+
assert.notEqual(cropped, null);
212+
assertCroppedPixels(cropped!, image, box);
213+
},
214+
),
215+
{ numRuns: PROPERTY_RUNS },
216+
);
217+
});
218+
219+
function claimable(width: number, height: number): PngFixture {
220+
return {
221+
pixels: rampPixels(width, height, 4, () => OPAQUE),
222+
width,
223+
height,
224+
channels: 4,
225+
colorType: 6,
226+
};
227+
}
228+
229+
function translucentAt(
230+
width: number,
231+
height: number,
232+
translucent: Readonly<{ x: number; y: number }>,
233+
): PngFixture {
234+
return {
235+
...claimable(width, height),
236+
pixels: rampPixels(width, height, 4, (x, y) =>
237+
x === translucent.x && y === translucent.y ? 128 : OPAQUE,
238+
),
239+
};
240+
}
241+
242+
function fullBox(width: number, height: number): Rect {
243+
return { x: 0, y: 0, width, height };
244+
}
245+
246+
function assertCroppedPixels(cropped: Buffer, image: PngFixture, box: Rect): void {
247+
const decoded = readPngForTest(cropped);
248+
assert.deepEqual([decoded.width, decoded.height], [box.width, box.height]);
249+
for (let row = 0; row < box.height; row += 1) {
250+
for (let column = 0; column < box.width; column += 1) {
251+
const from = ((row + box.y) * image.width + column + box.x) * image.channels;
252+
const to = (row * box.width + column) * 4;
253+
for (let channel = 0; channel < 3; channel += 1) {
254+
assert.equal(
255+
decoded.rgba[to + channel]!,
256+
image.pixels[from + channel]!,
257+
`pixel ${column},${row} channel ${channel}`,
258+
);
259+
}
260+
}
261+
}
262+
}

0 commit comments

Comments
 (0)