Skip to content

Commit 3962ae8

Browse files
juliusmarmingebompusclaude
authored
perf(desktop): picture-in-picture stops re-encoding an unchanged page (#20070)
Co-authored-by: Aaron Queen <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent e74f477 commit 3962ae8

2 files changed

Lines changed: 103 additions & 12 deletions

File tree

‎apps/desktop/src/preview/Manager.test.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,8 @@ const withManager = <A>(
329329

330330
interface TestCapturedPreviewImage {
331331
readonly toJPEG: () => Buffer;
332+
/** Only fixtures that deliver picture-in-picture frames need pixels to fingerprint. */
333+
readonly toBitmap?: () => Buffer;
332334
readonly getSize: () => { readonly width: number; readonly height: number };
333335
}
334336

@@ -3454,6 +3456,7 @@ describe("PreviewManager", () => {
34543456
const jpeg = Buffer.from("shared-preview-frame");
34553457
const capturePage = vi.fn(async () => ({
34563458
toJPEG: () => jpeg,
3459+
toBitmap: () => jpeg,
34573460
getSize: () => ({ width: 1280, height: 720 }),
34583461
}));
34593462
fromId.mockReturnValue({
@@ -3612,6 +3615,7 @@ describe("PreviewManager", () => {
36123615
const jpeg = Buffer.from("shared-preview-frame");
36133616
const capturePage = vi.fn(async () => ({
36143617
toJPEG: () => jpeg,
3618+
toBitmap: () => jpeg,
36153619
getSize: () => ({ width: 1280, height: 720 }),
36163620
}));
36173621
fromId.mockReturnValue(makeTestPreviewWebContents(capturePage));
@@ -3687,6 +3691,7 @@ describe("PreviewManager", () => {
36873691
const jpeg = Buffer.from("retry-preview-frame");
36883692
const capturePage = vi.fn(async () => ({
36893693
toJPEG: () => jpeg,
3694+
toBitmap: () => jpeg,
36903695
getSize: () => ({ width: 1280, height: 720 }),
36913696
}));
36923697
fromId.mockReturnValue(makeTestPreviewWebContents(capturePage));
@@ -3718,6 +3723,7 @@ describe("PreviewManager", () => {
37183723
const jpeg = Buffer.from("reloaded-preview-frame");
37193724
const capturePage = vi.fn(async () => ({
37203725
toJPEG: () => jpeg,
3726+
toBitmap: () => jpeg,
37213727
getSize: () => ({ width: 1280, height: 720 }),
37223728
}));
37233729
fromId.mockReturnValue(makeTestPreviewWebContents(capturePage));
@@ -3741,11 +3747,62 @@ describe("PreviewManager", () => {
37413747
),
37423748
);
37433749

3750+
effectIt.effect("encodes a picture-in-picture frame only when the page changes", () =>
3751+
withManager((manager) =>
3752+
Effect.gen(function* () {
3753+
let page = "static-page";
3754+
let pixelsReadable = true;
3755+
const toJPEG = vi.fn((pixels: string) => Buffer.from(`jpeg:${pixels}`));
3756+
const capturePage = vi.fn(async () => {
3757+
const pixels = page;
3758+
return {
3759+
toJPEG: () => toJPEG(pixels),
3760+
toBitmap: () => (pixelsReadable ? Buffer.from(pixels) : Buffer.alloc(0)),
3761+
getSize: () => ({ width: 1280, height: 720 }),
3762+
};
3763+
});
3764+
fromId.mockReturnValue(makeTestPreviewWebContents(capturePage));
3765+
const { pictureInPictureWindow, send } = makeTestPictureInPictureWindow();
3766+
browserWindowConstructor.mockImplementation(function () {
3767+
return pictureInPictureWindow;
3768+
});
3769+
3770+
yield* manager.createTab("tab_pip_unchanged");
3771+
yield* manager.registerWebview("tab_pip_unchanged", 42);
3772+
yield* manager.openPictureInPicture("tab_pip_unchanged");
3773+
yield* TestClock.adjust(100);
3774+
3775+
expect(capturePage).toHaveBeenCalledTimes(2);
3776+
expect(toJPEG).toHaveBeenCalledOnce();
3777+
expect(send).toHaveBeenCalledOnce();
3778+
3779+
page = "changed-page";
3780+
yield* TestClock.adjust(100);
3781+
3782+
expect(toJPEG).toHaveBeenCalledTimes(2);
3783+
expect(send).toHaveBeenCalledTimes(2);
3784+
expect(send).toHaveBeenLastCalledWith(
3785+
"desktop:preview-pip-frame",
3786+
expect.objectContaining({ data: Buffer.from("jpeg:changed-page").toString("base64") }),
3787+
);
3788+
3789+
// A frame whose pixels cannot be read has no fingerprint and is still delivered.
3790+
pixelsReadable = false;
3791+
yield* TestClock.adjust(100);
3792+
3793+
expect(toJPEG).toHaveBeenCalledTimes(3);
3794+
expect(send).toHaveBeenCalledTimes(3);
3795+
yield* manager.closePictureInPicture("tab_pip_unchanged");
3796+
}),
3797+
),
3798+
);
3799+
37443800
effectIt.effect("drops empty frames before picture-in-picture delivery", () =>
37453801
withManager((manager) =>
37463802
Effect.gen(function* () {
37473803
const validImage: TestCapturedPreviewImage = {
37483804
toJPEG: () => Buffer.from("valid-preview-frame"),
3805+
toBitmap: () => Buffer.from("valid-preview-frame"),
37493806
getSize: () => ({ width: 1280, height: 720 }),
37503807
};
37513808
const capturePage = vi.fn(async () => validImage);

‎apps/desktop/src/preview/Manager.ts‎

Lines changed: 46 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
* elements live in the renderer; we only attach listeners and forward state
66
* here). Single layer-scoped browser session partition.
77
*/
8+
import * as NodeZlib from "node:zlib";
89
import {
910
DesktopPreviewRecordingInputSchema,
1011
DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER,
@@ -318,6 +319,33 @@ const cropAnnotationScreenshot = (
318319
return { dataUrl: cropped.toDataURL(), width: size.width, height: size.height, cropRect };
319320
};
320321

322+
/**
323+
* Electron converts `toBitmap()` output to sRGB by default. Captures on P3 Macs are
324+
* P3, so that conversion costs more per tick than the JPEG encode the fingerprint
325+
* replaces. An invalid target color space has no Skia color space, so `readPixels`
326+
* copies the pixels without a color transform. The fingerprint only compares frames
327+
* with each other, so color accuracy is irrelevant.
328+
*/
329+
const FINGERPRINT_BITMAP_OPTIONS: Electron.ToBitmapOptions = {
330+
colorSpace: { primaries: "invalid", transfer: "invalid", matrix: "invalid", range: "invalid" },
331+
};
332+
333+
/**
334+
* Identity of a captured frame: cheap enough to compute every tick, unlike a JPEG
335+
* encode. Null when the pixels could not be read (Electron reports a failed
336+
* `readPixels` as an empty buffer), so the caller treats the frame as changed.
337+
*/
338+
const fingerprintCapturedFrame = (image: Electron.NativeImage, size: Electron.Size) => {
339+
let bitmap: Buffer;
340+
try {
341+
bitmap = image.toBitmap(FINGERPRINT_BITMAP_OPTIONS);
342+
} catch {
343+
return null;
344+
}
345+
if (bitmap.byteLength === 0) return null;
346+
return `${size.width}x${size.height}:${NodeZlib.crc32(bitmap)}`;
347+
};
348+
321349
const findZoomStep = (current: number): number => {
322350
const index = ZOOM_LEVELS.findIndex(
323351
(level) => Math.abs(level - current) < ZOOM_EPSILON || level > current,
@@ -370,7 +398,7 @@ interface FrameCaptureSession {
370398
readonly scope: Scope.Closeable | null;
371399
readonly consumers: ReadonlySet<FrameCaptureConsumer>;
372400
readonly unthrottledWebContentsIds: ReadonlySet<number>;
373-
readonly lastPictureInPictureFrame: Buffer | null;
401+
readonly lastPictureInPictureFingerprint: string | null;
374402
}
375403

376404
interface PictureInPictureSession {
@@ -793,8 +821,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
793821
...current,
794822
scope: consumer === "picture-in-picture" ? null : current.scope,
795823
consumers,
796-
lastPictureInPictureFrame:
797-
consumer === "picture-in-picture" ? null : current.lastPictureInPictureFrame,
824+
lastPictureInPictureFingerprint:
825+
consumer === "picture-in-picture"
826+
? null
827+
: current.lastPictureInPictureFingerprint,
798828
});
799829
}),
800830
] as const;
@@ -2584,6 +2614,16 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
25842614
) {
25852615
return;
25862616
}
2617+
const frameSession = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId);
2618+
if (frameSession?.scope !== captureSession.scope) return;
2619+
// Fingerprint the raw pixels before encoding so an unchanged page skips the
2620+
// synchronous JPEG encode, the base64 copy and the IPC send. An unreadable
2621+
// frame has no fingerprint and is always delivered.
2622+
const fingerprint = fingerprintCapturedFrame(image, size);
2623+
const pictureInPicture =
2624+
frameSession.consumers.has("picture-in-picture") &&
2625+
(fingerprint === null || frameSession.lastPictureInPictureFingerprint !== fingerprint);
2626+
if (!pictureInPicture) return;
25872627
const encoded = yield* attempt(
25882628
{
25892629
operation: "frameCapture.encodeFrame",
@@ -2592,12 +2632,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
25922632
},
25932633
() => image.toJPEG(PICTURE_IN_PICTURE_JPEG_QUALITY),
25942634
);
2595-
const frameSession = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId);
2596-
if (frameSession?.scope !== captureSession.scope) return;
2597-
const pictureInPicture =
2598-
frameSession.consumers.has("picture-in-picture") &&
2599-
frameSession.lastPictureInPictureFrame?.equals(encoded) !== true;
2600-
if (!pictureInPicture) return;
26012635
const receivedAt = yield* currentIso;
26022636
const frame: DesktopPreviewRecordingFrame = {
26032637
tabId,
@@ -2662,7 +2696,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
26622696
return replaceMap(sessions, (copy) => {
26632697
copy.set(tabId, {
26642698
...frameSession,
2665-
lastPictureInPictureFrame: encoded,
2699+
lastPictureInPictureFingerprint: fingerprint,
26662700
});
26672701
});
26682702
});
@@ -2754,7 +2788,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
27542788
scope,
27552789
consumers: new Set([consumer]),
27562790
unthrottledWebContentsIds: new Set([wc.id]),
2757-
lastPictureInPictureFrame: null,
2791+
lastPictureInPictureFingerprint: null,
27582792
});
27592793
}),
27602794
] as const;
@@ -2920,7 +2954,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
29202954
const current = sessions.get(tabId);
29212955
if (!current?.consumers.has("picture-in-picture")) return sessions;
29222956
return replaceMap(sessions, (copy) => {
2923-
copy.set(tabId, { ...current, lastPictureInPictureFrame: null });
2957+
copy.set(tabId, { ...current, lastPictureInPictureFingerprint: null });
29242958
});
29252959
}),
29262960
);

0 commit comments

Comments
 (0)