Skip to content

Commit c0df8ee

Browse files
committed
Merge remote-tracking branch 'origin/main' into rebase/maestro-setpermissions
* origin/main: feat(diff): accept JPEG inputs for screenshot comparison (#2718) refactor(system-button): declare home, app-switcher and action-button as one button family (#2715) # Conflicts: # CHANGELOG.md
2 parents 55466f9 + be0902e commit c0df8ee

70 files changed

Lines changed: 1127 additions & 1014 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎CHANGELOG.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,15 @@
22

33
## Unreleased
44

5+
- Added (diff): `diff screenshot` accepts a JPEG baseline or current image. Both inputs had to be PNG,
6+
so a capture exported by another tool had to be converted first and a HarmonyOS capture — which the
7+
platform serves as JPEG under whatever name the command was given — could never be compared. Each
8+
input is now decoded from its own bytes, so the container is sniffed and a `.png` name holding JPEG
9+
decodes as JPEG. `png-transcode.ts` became `screenshot-image.ts`, the one owner of that sniffing for
10+
both the decode and the provider transcode path, and the PNG worker gained a `decode-image` job that
11+
answers pixels instead of PNG bytes. The `--out` diff image stays PNG, as do the crop, overlay, and
12+
resize passes that rewrite a screenshot in place and could not survive a lossy container.
13+
514
- Added (maestro): `setPermissions` and `launchApp.permissions` support `all: allow|deny|unset`
615
on iOS simulators and Android, with specific entries overriding `all`. Both accept a
716
Maestro-style permissions map. Entries apply in order until the selected platform refuses one,
@@ -33,6 +42,12 @@
3342
none, and Android, HarmonyOS, Vega, Linux, web, tvOS, macOS, and visionOS each state their own refusal.
3443
Simulators run no Shortcuts or App Intents, so what a press triggers is verifiable only on a physical
3544
iPhone (#2699).
45+
- Changed (all): `home`, `app-switcher` and `action-button` are one system-button family in the
46+
runtime contract. An owner without a button now refuses it with the family's hint (for example
47+
`Android has no key event for this system button.`) rather than a per-button sentence, and a
48+
Limrun or WebDriver session that is no longer active refuses `action-button` with that session's
49+
own reason like every other cell; the command name still leads the `UNSUPPORTED_OPERATION`
50+
message.
3651
- Fixed (android): `clipboard read` and `clipboard write` stop reporting success on a build whose
3752
clipboard service has no shell command. Android 16 (API 36) answers every `adb shell cmd clipboard …`
3853
with the framework default `Binder.handleShellCommand` — `No shell command implementation.` on

‎packages/capture-kit/src/png-worker-client.test.ts‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import {
66
computePngRgbDifferenceAsync,
77
computeScreenshotDiffPixelsAsync,
88
decodePngAsync,
9+
decodeScreenshotImageAsync,
910
encodePngAsync,
1011
terminatePngWorker,
1112
transcodeScreenshotToPngAsync,
@@ -110,7 +111,7 @@ test('decodePngAsync rejects invalid PNG data with the canonical decode AppError
110111

111112
test('transcodeScreenshotToPngAsync matches the synchronous transcoder byte for byte', async () => {
112113
const { encode } = await import('jpeg-js');
113-
const { transcodeScreenshotToPng } = await import('./png-transcode.ts');
114+
const { transcodeScreenshotToPng } = await import('./screenshot-image.ts');
114115
const rgba = Buffer.alloc(9 * 7 * 4, 0x66);
115116
const jpeg = encode({ width: 9, height: 7, data: rgba }, 90).data;
116117

@@ -138,3 +139,50 @@ test('transcodeScreenshotToPngAsync rejects a corrupt JPEG with the canonical de
138139
return true;
139140
});
140141
});
142+
143+
test('decodeScreenshotImageAsync answers the synchronous decoder for both containers', async () => {
144+
const { encode } = await import('jpeg-js');
145+
const { decodeScreenshotImage } = await import('./screenshot-image.ts');
146+
const png = PNG.sync.write(buildPatternPng(11, 8, 4));
147+
const jpeg = encode({ width: 11, height: 8, data: buildPatternPng(11, 8, 4).data }, 90).data;
148+
149+
for (const bytes of [png, jpeg]) {
150+
const fromWorker = await decodeScreenshotImageAsync(bytes, 'fixture');
151+
const fromSync = decodeScreenshotImage(bytes, 'fixture');
152+
153+
assert.deepEqual([fromWorker.width, fromWorker.height], [fromSync.width, fromSync.height]);
154+
assert.deepEqual(fromWorker.data, fromSync.data);
155+
}
156+
});
157+
158+
test('decodeScreenshotImageAsync decodes a JPEG by its bytes, not its label', async () => {
159+
const { encode } = await import('jpeg-js');
160+
const jpeg = encode({ width: 9, height: 7, data: buildPatternPng(9, 7, 5).data }, 90).data;
161+
162+
const decoded = await decodeScreenshotImageAsync(jpeg, 'fixture.png');
163+
164+
assert.deepEqual([decoded.width, decoded.height], [9, 7]);
165+
assert.equal(decoded.data.length, 9 * 7 * 4);
166+
});
167+
168+
test('decodeScreenshotImageAsync rejects a container it cannot read with the canonical AppError', async () => {
169+
await assert.rejects(
170+
() => decodeScreenshotImageAsync(Buffer.from('GIF89a'), 'fixture'),
171+
(error) => {
172+
assert.equal(error instanceof AppError, true);
173+
assert.equal((error as AppError).code, 'COMMAND_FAILED');
174+
assert.equal((error as AppError).message, 'fixture is neither PNG nor JPEG');
175+
assert.equal((error as AppError).details?.label, 'fixture');
176+
return true;
177+
},
178+
);
179+
180+
const corrupt = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(16, 0)]);
181+
await assert.rejects(
182+
() => decodeScreenshotImageAsync(corrupt, 'fixture'),
183+
(error) => {
184+
assert.match((error as AppError).message, /Failed to decode fixture as JPEG/);
185+
return true;
186+
},
187+
);
188+
});

‎packages/capture-kit/src/png-worker-client.ts‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,31 @@ export async function decodePngAsync(buffer: Buffer, label: string): Promise<PNG
201201
const png = decodePng(buffer, label);
202202
return { kind: 'decode', width: png.width, height: png.height, data: png.data };
203203
});
204+
return toDecodedPng(result);
205+
}
206+
207+
/**
208+
* Decodes a screenshot in whatever container it arrived in into RGBA pixels, sniffing the container
209+
* instead of trusting a file name. Use it where a command reads an image someone else produced; a
210+
* command that rewrites a screenshot in place stays on `decodePngAsync`, which keeps the file's
211+
* container honest. Decode failures carry the canonical `AppError`.
212+
*/
213+
export async function decodeScreenshotImageAsync(bytes: Buffer, label: string): Promise<PNG> {
214+
const result = await runPngJob({ kind: 'decode-image', image: bytes, label }, async () => {
215+
// Read on demand so the JPEG decoder stays out of the import closure of every entry that only
216+
// needs the worker's other jobs.
217+
const { decodeScreenshotImage } = await import('./screenshot-image.ts');
218+
const image = decodeScreenshotImage(bytes, label);
219+
return { kind: 'decode-image', width: image.width, height: image.height, data: image.data };
220+
});
221+
return toDecodedPng(result);
222+
}
223+
224+
/**
225+
* Rebuilds the decoded-image handle on this side of the worker boundary. `PNG` is the shape every
226+
* pixel reader here consumes: size plus RGBA rows.
227+
*/
228+
function toDecodedPng(result: { width: number; height: number; data: Uint8Array }): PNG {
204229
const png = new PNG({ width: result.width, height: result.height });
205230
png.data = toBuffer(result.data);
206231
return png;
@@ -267,7 +292,7 @@ export async function transcodeScreenshotToPngAsync(bytes: Buffer, label: string
267292
const result = await runPngJob({ kind: 'jpeg-to-png', image: bytes, label }, async () => {
268293
// Read on demand so the JPEG decoder stays out of the import closure of every entry that only
269294
// needs the worker's other jobs.
270-
const { transcodeScreenshotToPng } = await import('./png-transcode.ts');
295+
const { transcodeScreenshotToPng } = await import('./screenshot-image.ts');
271296
return { kind: 'jpeg-to-png', png: transcodeScreenshotToPng(bytes, label) };
272297
});
273298
return toBuffer(result.png);

‎packages/capture-kit/src/png-worker-contract.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ import type { PngRgbDifferenceResult } from './png-rgb-difference.ts';
1616

1717
export type PngWorkerJob =
1818
| { kind: 'decode'; png: Uint8Array; label: string }
19+
// A screenshot in whatever container it arrived in; answers decoded pixels, not PNG bytes.
20+
| { kind: 'decode-image'; image: Uint8Array; label: string }
1921
| { kind: 'encode'; width: number; height: number; data: Uint8Array }
2022
| { kind: 'crop'; png: Uint8Array; label: string; box: Rect }
2123
| { kind: 'rgb-difference'; firstPng: Uint8Array; secondPng: Uint8Array; label: string }
@@ -25,6 +27,7 @@ export type PngWorkerJob =
2527

2628
export type PngWorkerJobResult =
2729
| { kind: 'decode'; width: number; height: number; data: Uint8Array }
30+
| { kind: 'decode-image'; width: number; height: number; data: Uint8Array }
2831
| { kind: 'encode'; png: Uint8Array }
2932
// A crop answers `null` when the box already covers the image, so the caller keeps the file.
3033
| { kind: 'crop'; png: Uint8Array | null }

‎packages/capture-kit/src/png-worker.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,18 @@ test('resultTransferList transfers a cropped encoding and skips an untouched fil
2828
assert.deepEqual(resultTransferList({ kind: 'crop', png: null }), []);
2929
});
3030

31+
test('resultTransferList transfers decoded pixels for both screenshot decode jobs', () => {
32+
const owned = Buffer.alloc(16); // Buffer.alloc never uses the shared pool
33+
const pooled = new Uint8Array(new ArrayBuffer(32), 4, 8); // offset view, pooled-Buffer shape
34+
const decoded = { width: 2, height: 2 };
35+
36+
assert.deepEqual(resultTransferList({ kind: 'decode', ...decoded, data: owned }), [owned.buffer]);
37+
assert.deepEqual(resultTransferList({ kind: 'decode-image', ...decoded, data: owned }), [
38+
owned.buffer,
39+
]);
40+
assert.deepEqual(resultTransferList({ kind: 'decode-image', ...decoded, data: pooled }), []);
41+
});
42+
3143
test('resultTransferList transfers only the fully-owned views of a mixed result', () => {
3244
const ownedDiffData = Buffer.alloc(16); // Buffer.alloc never uses the shared pool
3345
const pooledMask = new Uint8Array(new ArrayBuffer(32), 4, 8); // offset view, pooled-Buffer shape

‎packages/capture-kit/src/png-worker.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,19 @@ import {
1717
* `png-worker-client.ts`; published as the `internal/png-worker` build entry.
1818
*/
1919

20-
// The daemon prewarms this worker at startup, so the JPEG decoder loads only for a transcode job.
20+
// The daemon prewarms this worker at startup, so the JPEG decoder loads only for a job that can
21+
// carry JPEG bytes.
2122
async function runJob(request: PngWorkerRequest): Promise<PngWorkerJobResult> {
2223
switch (request.kind) {
2324
case 'decode': {
2425
const png = decodePng(toBuffer(request.png), request.label);
2526
return { kind: 'decode', width: png.width, height: png.height, data: png.data };
2627
}
28+
case 'decode-image': {
29+
const { decodeScreenshotImage } = await import('./screenshot-image.ts');
30+
const image = decodeScreenshotImage(toBuffer(request.image), request.label);
31+
return { kind: 'decode-image', width: image.width, height: image.height, data: image.data };
32+
}
2733
case 'encode': {
2834
const png = new PNG({ width: request.width, height: request.height });
2935
png.data = toBuffer(request.data);
@@ -44,7 +50,7 @@ async function runJob(request: PngWorkerRequest): Promise<PngWorkerJobResult> {
4450
return { kind: 'diff-pixels', ...computeScreenshotDiffPixels(request) };
4551
}
4652
case 'jpeg-to-png': {
47-
const { transcodeScreenshotToPng } = await import('./png-transcode.ts');
53+
const { transcodeScreenshotToPng } = await import('./screenshot-image.ts');
4854
return {
4955
kind: 'jpeg-to-png',
5056
png: transcodeScreenshotToPng(toBuffer(request.image), request.label),
@@ -80,6 +86,7 @@ export function resultTransferList(result: PngWorkerJobResult): ArrayBuffer[] {
8086
function resultBufferViews(result: PngWorkerJobResult): Uint8Array[] {
8187
switch (result.kind) {
8288
case 'decode':
89+
case 'decode-image':
8390
return [result.data];
8491
case 'encode':
8592
return [result.png];

packages/capture-kit/src/png-transcode.test.ts renamed to packages/capture-kit/src/screenshot-image.test.ts

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
import { expect, test } from 'vitest';
22
import { encode as encodeJpeg } from 'jpeg-js';
33
import { PNG } from './png.ts';
4-
import { detectScreenshotImageFormat, transcodeScreenshotToPng } from './png-transcode.ts';
4+
import {
5+
decodeScreenshotImage,
6+
detectScreenshotImageFormat,
7+
transcodeScreenshotToPng,
8+
} from './screenshot-image.ts';
59

610
function solidRgba(width: number, height: number, rgba: readonly [number, number, number, number]) {
711
const data = Buffer.alloc(width * height * 4);
@@ -74,3 +78,43 @@ test('a truncated JPEG body is refused with the same typed decode error', () =>
7478
message: 'Failed to decode Limrun iOS screenshot as JPEG',
7579
});
7680
});
81+
82+
test('decoding a PNG answers its pixels without re-encoding the container', () => {
83+
const png = new PNG({ width: 3, height: 2 });
84+
png.data = solidRgba(3, 2, [12, 240, 60, 255]);
85+
86+
const decoded = decodeScreenshotImage(PNG.sync.write(png), 'test screenshot');
87+
88+
expect([decoded.width, decoded.height]).toEqual([3, 2]);
89+
expect([...decoded.data.subarray(0, 4)]).toEqual([12, 240, 60, 255]);
90+
});
91+
92+
test('decoding a JPEG answers RGBA rows, with opaque alpha where the container has none', () => {
93+
const jpeg = encodeJpeg({ width: 5, height: 3, data: solidRgba(5, 3, [12, 240, 60, 255]) }, 100);
94+
95+
const decoded = decodeScreenshotImage(jpeg.data, 'test screenshot');
96+
97+
expect([decoded.width, decoded.height]).toEqual([5, 3]);
98+
expect(decoded.data.length).toBe(5 * 3 * 4);
99+
const [r = -1, g = -1, b = -1, a = -1] = decoded.data.subarray(0, 4);
100+
// JPEG is lossy; a flat field survives within a few levels per channel.
101+
expect(Math.abs(r - 12)).toBeLessThanOrEqual(4);
102+
expect(Math.abs(g - 240)).toBeLessThanOrEqual(4);
103+
expect(Math.abs(b - 60)).toBeLessThanOrEqual(4);
104+
expect(a).toBe(255);
105+
});
106+
107+
test('decoding refuses bytes in neither container and a JPEG body that cannot be decoded', () => {
108+
expect(thrownBy(() => decodeScreenshotImage(Buffer.from('GIF89a'), 'fixture'))).toMatchObject({
109+
code: 'COMMAND_FAILED',
110+
message: 'fixture is neither PNG nor JPEG',
111+
details: { label: 'fixture', leadingBytes: '47494638' },
112+
});
113+
114+
const corrupt = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(32, 0x41)]);
115+
expect(thrownBy(() => decodeScreenshotImage(corrupt, 'fixture'))).toMatchObject({
116+
code: 'COMMAND_FAILED',
117+
message: 'Failed to decode fixture as JPEG',
118+
details: { label: 'fixture', reason: expect.any(String) },
119+
});
120+
});
Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { AppError } from '@agent-device/kernel/errors';
22
import { decode as decodeJpeg } from 'jpeg-js';
3-
import { hasPngSignature, PNG } from './png.ts';
3+
import { decodePng, hasPngSignature, PNG } from './png.ts';
44

55
const JPEG_SIGNATURE = Buffer.from([0xff, 0xd8, 0xff]);
66

@@ -13,35 +13,55 @@ export function detectScreenshotImageFormat(bytes: Buffer): ScreenshotImageForma
1313
return undefined;
1414
}
1515

16+
/**
17+
* Decodes a screenshot in whatever container it arrived in into the RGBA rows every pixel reader in
18+
* this repository consumes. The container is sniffed, never trusted from a file name or a caller's
19+
* guess, so a JPEG stored under a `.png` name still decodes.
20+
*
21+
* Synchronous and CPU-bound: daemon request paths reach it through the PNG worker
22+
* (`decodeScreenshotImageAsync` in `png-worker-client.ts`), like every other codec job.
23+
*/
24+
export function decodeScreenshotImage(bytes: Buffer, label: string): PNG {
25+
const format = detectScreenshotImageFormat(bytes);
26+
if (format === 'png') return decodePng(bytes, label);
27+
if (format === 'jpeg') return decodeJpegImage(bytes, label);
28+
throw unsupportedContainer(label, bytes);
29+
}
30+
1631
/**
1732
* Returns PNG bytes for a screenshot a provider handed back in whatever container it prefers. PNG
1833
* passes through untouched; JPEG is decoded and re-encoded losslessly from the decoded pixels, so
19-
* every PNG-only reader downstream (size, crop, overlay, diff) sees the format the path promises.
34+
* every PNG-only reader downstream (size, crop, overlay) sees the format the path promises. Those
35+
* readers rewrite the file in place, which is why a JPEG cannot be handed to them undecoded.
2036
*
2137
* Synchronous and CPU-bound: daemon request paths reach it through the PNG worker
2238
* (`transcodeScreenshotToPngAsync` in `png-worker-client.ts`), like every other codec job.
2339
*/
2440
export function transcodeScreenshotToPng(bytes: Buffer, label: string): Buffer {
2541
const format = detectScreenshotImageFormat(bytes);
2642
if (format === 'png') return bytes;
27-
if (format === 'jpeg') return jpegToPng(bytes, label);
28-
throw new AppError('COMMAND_FAILED', `${label} is neither PNG nor JPEG`, {
29-
label,
30-
leadingBytes: bytes.subarray(0, 4).toString('hex'),
31-
});
43+
if (format === 'jpeg') return PNG.sync.write(decodeJpegImage(bytes, label));
44+
throw unsupportedContainer(label, bytes);
3245
}
3346

3447
/** A JPEG signature does not prove a decodable body; a failure keeps the label and the decoder's reason. */
35-
function jpegToPng(bytes: Buffer, label: string): Buffer {
48+
function decodeJpegImage(bytes: Buffer, label: string): PNG {
3649
try {
3750
const decoded = decodeJpeg(bytes, { useTArray: true, formatAsRGBA: true });
3851
const png = new PNG({ width: decoded.width, height: decoded.height });
3952
png.data = Buffer.from(decoded.data.buffer, decoded.data.byteOffset, decoded.data.byteLength);
40-
return PNG.sync.write(png);
53+
return png;
4154
} catch (error) {
4255
throw new AppError('COMMAND_FAILED', `Failed to decode ${label} as JPEG`, {
4356
label,
4457
reason: error instanceof Error ? error.message : String(error),
4558
});
4659
}
4760
}
61+
62+
function unsupportedContainer(label: string, bytes: Buffer): AppError {
63+
return new AppError('COMMAND_FAILED', `${label} is neither PNG nor JPEG`, {
64+
label,
65+
leadingBytes: bytes.subarray(0, 4).toString('hex'),
66+
});
67+
}

‎packages/contracts/package.json‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,6 @@
6464
"types": "./src/app-state-runtime.ts",
6565
"default": "./src/app-state-runtime.ts"
6666
},
67-
"./app-switcher-runtime": {
68-
"types": "./src/app-switcher-runtime.ts",
69-
"default": "./src/app-switcher-runtime.ts"
70-
},
7167
"./apple-runner-request": {
7268
"types": "./src/apple-runner-request.ts",
7369
"default": "./src/apple-runner-request.ts"
@@ -220,10 +216,6 @@
220216
"types": "./src/gesture-runtime.ts",
221217
"default": "./src/gesture-runtime.ts"
222218
},
223-
"./home-runtime": {
224-
"types": "./src/home-runtime.ts",
225-
"default": "./src/home-runtime.ts"
226-
},
227219
"./host-diagnostics": {
228220
"types": "./src/host-diagnostics.ts",
229221
"default": "./src/host-diagnostics.ts"
@@ -456,6 +448,10 @@
456448
"types": "./src/startup-recovery-fence.ts",
457449
"default": "./src/startup-recovery-fence.ts"
458450
},
451+
"./system-button-runtime": {
452+
"types": "./src/system-button-runtime.ts",
453+
"default": "./src/system-button-runtime.ts"
454+
},
459455
"./tap-keyboard-occlusion": {
460456
"types": "./src/tap-keyboard-occlusion.ts",
461457
"default": "./src/tap-keyboard-occlusion.ts"

0 commit comments

Comments
 (0)