Skip to content

Commit d50a6fd

Browse files
MansTombjuliusmarmingeclaudemaria-rcks
authored
fix(web): preview recordings stay decodable after a viewport resize (#10495)
Co-authored-by: Julius Marminge <[email protected]> Co-authored-by: Claude Fable 5.1 <[email protected]> Co-authored-by: maria-rcks <[email protected]>
1 parent 0af2d21 commit d50a6fd

4 files changed

Lines changed: 89 additions & 8 deletions

File tree

‎apps/web/src/browser/browserRecording.test.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,10 @@ vi.mock(import("~/components/preview/previewBridge"), async (original) => {
6969
}
7070
});
7171

72+
vi.mock("./recordingCompositor", () => ({
73+
createRecordingCompositor: async (stream: MediaStream) => ({ stream, dispose: () => {} }),
74+
}));
75+
7276
vi.mock("~/rpc/atomRegistry", () => ({
7377
appAtomRegistry: { set: registrySet },
7478
}));

‎apps/web/src/browser/browserRecording.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ interface ActiveRecording {
144144
releaseSurfaceActivity: (() => void) | null;
145145
stream: MediaStream | null;
146146
recorder: MediaRecorder | null;
147-
compositor: Awaited<ReturnType<typeof createRecordingCompositor>>;
147+
compositor: Awaited<ReturnType<typeof createRecordingCompositor>> | null;
148148
lifecycle: BrowserRecordingLifecycle;
149149
}
150150

@@ -641,7 +641,7 @@ export async function startBrowserRecording(
641641
if (event.tabId === tabId) listener(event.input);
642642
}),
643643
);
644-
recorder = createMediaRecorder(recording.compositor?.stream ?? stream);
644+
recorder = createMediaRecorder(recording.compositor.stream);
645645
recording.recorder = recorder;
646646
recorder.addEventListener("dataavailable", (event) => {
647647
if (event.data.size > 0) chunks.push(event.data);
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import { describe, expect, it, vi } from "vite-plus/test";
2+
3+
import { createRecordingCompositor } from "./recordingCompositor";
4+
5+
const paintSource = (canvas: HTMLCanvasElement, color: string) => {
6+
const context = canvas.getContext("2d");
7+
if (!context) throw new Error("Source canvas is unavailable.");
8+
context.fillStyle = color;
9+
context.fillRect(0, 0, canvas.width, canvas.height);
10+
};
11+
12+
const samplePixel = (video: HTMLVideoElement, x: number, y: number) => {
13+
const canvas = document.createElement("canvas");
14+
canvas.width = video.videoWidth;
15+
canvas.height = video.videoHeight;
16+
const context = canvas.getContext("2d", { willReadFrequently: true });
17+
if (!context) throw new Error("Sample canvas is unavailable.");
18+
context.drawImage(video, 0, 0);
19+
return Array.from(context.getImageData(x, y, 1, 1).data.slice(0, 3));
20+
};
21+
22+
describe("recording compositor", () => {
23+
it("keeps the starting size and letterboxes a resized source", async () => {
24+
const source = document.createElement("canvas");
25+
source.width = 320;
26+
source.height = 180;
27+
paintSource(source, "#f00");
28+
const sourceStream = source.captureStream(30);
29+
const compositor = await createRecordingCompositor(
30+
sourceStream,
31+
{ showKeyPresses: false, showMousePresses: false, frameRate: 30 },
32+
() => () => {},
33+
);
34+
const output = document.createElement("video");
35+
output.muted = true;
36+
output.srcObject = compositor.stream;
37+
try {
38+
await output.play();
39+
await vi.waitFor(() => {
40+
expect([output.videoWidth, output.videoHeight]).toEqual([320, 180]);
41+
expect(samplePixel(output, 160, 90)).toEqual([255, 0, 0]);
42+
});
43+
44+
source.width = 90;
45+
source.height = 160;
46+
paintSource(source, "#0f0");
47+
48+
await vi.waitFor(() => {
49+
paintSource(source, "#0f0");
50+
expect(samplePixel(output, 160, 90)).toEqual([0, 255, 0]);
51+
});
52+
expect([output.videoWidth, output.videoHeight]).toEqual([320, 180]);
53+
expect(samplePixel(output, 10, 90)).toEqual([0, 0, 0]);
54+
} finally {
55+
output.srcObject = null;
56+
compositor.dispose();
57+
for (const track of sourceStream.getTracks()) track.stop();
58+
}
59+
});
60+
});

‎apps/web/src/browser/recordingCompositor.ts‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ export async function createRecordingCompositor(
1313
options: RecordingDecorationOptions,
1414
subscribe: (listener: (input: DesktopPreviewRecordingInput) => void) => () => void,
1515
) {
16-
if (!options.showKeyPresses && !options.showMousePresses) return null;
1716
const canvas = document.createElement("canvas");
1817
const context = canvas.getContext("2d", { alpha: false });
1918
if (!context) throw new Error("Recording canvas is unavailable.");
@@ -26,17 +25,35 @@ export async function createRecordingCompositor(
2625
canvas.height = settings?.height ?? 1080;
2726
const decorations = new RecordingDecorations(options, readPreviewAnnotationTheme().primary);
2827
let disposed = false;
28+
let sized = false;
2929
let frameId: number | undefined;
3030
let timer: number | undefined;
3131
const draw = () => {
3232
if (disposed || video.readyState < 2) return;
33-
const width = video.videoWidth || canvas.width;
34-
const height = video.videoHeight || canvas.height;
35-
if (canvas.width !== width) canvas.width = width;
36-
if (canvas.height !== height) canvas.height = height;
37-
context.drawImage(video, 0, 0, width, height);
33+
// The encoded size stays at the first frame's: avc1 MP4 stores one set of decoder
34+
// parameters, so a resized source is letterboxed instead of changing the stream size.
35+
if (!sized) {
36+
canvas.width = video.videoWidth || canvas.width;
37+
canvas.height = video.videoHeight || canvas.height;
38+
sized = true;
39+
}
40+
const sourceWidth = video.videoWidth || canvas.width;
41+
const sourceHeight = video.videoHeight || canvas.height;
42+
const fit = Math.min(canvas.width / sourceWidth, canvas.height / sourceHeight);
43+
const width = Math.round(sourceWidth * fit);
44+
const height = Math.round(sourceHeight * fit);
45+
const left = Math.floor((canvas.width - width) / 2);
46+
const top = Math.floor((canvas.height - height) / 2);
47+
if (width < canvas.width || height < canvas.height) {
48+
context.fillStyle = "#000";
49+
context.fillRect(0, 0, canvas.width, canvas.height);
50+
}
51+
context.drawImage(video, left, top, width, height);
3852
const now = performance.now();
53+
context.save();
54+
context.translate(left, top);
3955
decorations.draw(context, width, height, now);
56+
context.restore();
4057
window.clearTimeout(timer);
4158
const next = decorations.nextRedraw(now);
4259
if (next !== null) timer = window.setTimeout(draw, next);

0 commit comments

Comments
 (0)