Skip to content

Commit 70bd962

Browse files
fix(server): agent recordings honor the browser frame rate setting (pingdotgg#23895)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 5b55036 commit 70bd962

6 files changed

Lines changed: 112 additions & 12 deletions

File tree

‎apps/server/src/preview/ServerBrowser.test.ts‎

Lines changed: 77 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
import * as NodeEvents from "node:events";
2+
import * as NodeVM from "node:vm";
23
// @effect-diagnostics-next-line nodeBuiltinImport:off - Playwright download mocks perform real file operations outside the Effect runtime.
34
import * as NodeFSP from "node:fs/promises";
45

56
import * as NodeServices from "@effect/platform-node/NodeServices";
67
import * as HostProcess from "@t3tools/shared/HostProcess";
78
import { it } from "@effect/vitest";
89
import {
10+
type BrowserRecordingFrameRate,
911
EnvironmentId,
1012
PreviewTabId,
1113
ProviderInstanceId,
@@ -49,7 +51,7 @@ vi.mock("./ServerBrowserContexts.ts", () => ({
4951
return context as unknown as BrowserContext;
5052
}
5153
async scratchPage() {
52-
return makeContext().page as unknown as Page;
54+
return makeEncoderPage() as unknown as Page;
5355
}
5456
async connectDesktopPage(endpoint: string) {
5557
const context = makeContext();
@@ -212,7 +214,50 @@ function makeContext(onClose?: (context: BrowserContext) => void) {
212214
return context;
213215
}
214216

217+
// Runs the real encoder script against fake canvas and MediaRecorder globals.
218+
function makeEncoderPage() {
219+
const canvas = {
220+
width: 0,
221+
height: 0,
222+
getContext: () => ({ drawImage() {}, fillRect() {} }),
223+
captureStream: (frameRate?: number) => ({ frameRate, getTracks: () => [] }),
224+
};
225+
class FakeMediaRecorder {
226+
static isTypeSupported = (type: string) => type === "video/webm";
227+
onstart: (() => void) | null = null;
228+
constructor(stream: { frameRate?: number }, options: { videoBitsPerSecond: number }) {
229+
encoderStreams.push({
230+
frameRate: stream.frameRate,
231+
bitsPerSecond: options.videoBitsPerSecond,
232+
});
233+
}
234+
start() {
235+
this.onstart?.();
236+
}
237+
}
238+
const sandbox: Record<string, unknown> = {
239+
document: { createElement: () => canvas },
240+
Path2D: function Path2D() {},
241+
MediaRecorder: FakeMediaRecorder,
242+
createImageBitmap: async () => ({ width: 1280, height: 800, close() {} }),
243+
atob,
244+
Blob,
245+
performance,
246+
};
247+
sandbox.window = sandbox;
248+
const context = NodeVM.createContext(sandbox);
249+
return {
250+
...makeContext().page,
251+
evaluate: vi.fn(async (script: string | ((arg: unknown) => unknown), arg?: unknown) =>
252+
typeof script === "string"
253+
? NodeVM.runInContext(script, context)
254+
: NodeVM.runInContext(`(${script.toString()})`, context)(arg),
255+
),
256+
};
257+
}
258+
215259
const contexts: ReturnType<typeof makeContext>[] = [];
260+
const encoderStreams: Array<{ frameRate: number | undefined; bitsPerSecond: number }> = [];
216261
const contextRequests: Array<{ profileId: string; isolated: boolean }> = [];
217262
let contextGate: PromiseWithResolvers<void> | null = null;
218263
let pageCopy = "";
@@ -320,6 +365,7 @@ const viewerInput = (tabId: string, canOperate: boolean) => ({
320365

321366
beforeEach(() => {
322367
contexts.length = 0;
368+
encoderStreams.length = 0;
323369
contextRequests.length = 0;
324370
contextGate = null;
325371
contextFailure = null;
@@ -418,6 +464,36 @@ it.live.each([
418464
).pipe(Effect.provide(layer)),
419465
);
420466

467+
// The fake encoder decodes every frame at 1280x800, so 30 fps hits the bitrate floor.
468+
it.live.each([
469+
{ reports: [], frameRate: 30, bitsPerSecond: 2.5e6 },
470+
{ reports: [30], frameRate: 30, bitsPerSecond: 2.5e6 },
471+
{ reports: [60], frameRate: 60, bitsPerSecond: 1280 * 800 * 60 * 0.05 },
472+
{ reports: [60, undefined], frameRate: 60, bitsPerSecond: 1280 * 800 * 60 * 0.05 },
473+
{ reports: [60, 30], frameRate: 30, bitsPerSecond: 2.5e6 },
474+
] satisfies ReadonlyArray<{
475+
reports: ReadonlyArray<BrowserRecordingFrameRate | undefined>;
476+
frameRate: number;
477+
bitsPerSecond: number;
478+
}>)(
479+
"agent recordings encode at the last reported frame rate ($reports)",
480+
({ reports, frameRate, bitsPerSecond }) =>
481+
Effect.scoped(
482+
Effect.gen(function* () {
483+
const { browser, broker, tabId } = yield* ready;
484+
for (const recordingFrameRate of reports) {
485+
yield* browser.reportProfiles({
486+
profiles: [],
487+
defaultProfileId: "default",
488+
...(recordingFrameRate === undefined ? {} : { recordingFrameRate }),
489+
});
490+
}
491+
yield* broker.invoke({ scope, tabId, operation: "recordingStart", input: {} });
492+
expect(encoderStreams).toEqual([{ frameRate, bitsPerSecond }]);
493+
}),
494+
).pipe(Effect.provide(layer)),
495+
);
496+
421497
it.live("enforces provider ownership and explicit targets when a session has multiple tabs", () =>
422498
Effect.scoped(
423499
Effect.gen(function* () {

‎apps/server/src/preview/ServerBrowser.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import {
55
BUILT_IN_BROWSER_PROFILES,
66
DEFAULT_BROWSER_PROFILE_ID,
7+
DEFAULT_BROWSER_RECORDING_FRAME_RATE,
78
FILL_PREVIEW_VIEWPORT,
89
findBrowserProfile,
910
INCOGNITO_BROWSER_PROFILE_ID,
@@ -1313,9 +1314,13 @@ const make = Effect.gen(function* () {
13131314
readonly profiles: ReadonlyArray<BrowserProfile>;
13141315
readonly defaultProfileId: string;
13151316
} | null = null;
1317+
// Agent recordings encode here, so they need the client's Browser recording frame rate. The
1318+
// last report outlives the client, which lets agents record while no client is connected.
1319+
let recordingFrameRate = DEFAULT_BROWSER_RECORDING_FRAME_RATE;
13161320

13171321
const reportProfiles: ServerBrowser["Service"]["reportProfiles"] = (input) =>
13181322
Effect.sync(() => {
1323+
if (input.recordingFrameRate !== undefined) recordingFrameRate = input.recordingFrameRate;
13191324
const profiles = resolveBrowserProfiles(input.profiles);
13201325
reportedProfiles = {
13211326
profiles,
@@ -1472,7 +1477,7 @@ const make = Effect.gen(function* () {
14721477
const encoder = await contexts.scratchPage();
14731478
let session: CDPSession | null = null;
14741479
try {
1475-
await encoder.evaluate(ServerBrowserPage.RECORDING_ENCODER_SCRIPT);
1480+
await encoder.evaluate(ServerBrowserPage.recordingEncoderScript(recordingFrameRate));
14761481
// Seed idle pages before returning so an immediate stop has a frame to encode.
14771482
const firstFrame = await ServerBrowserPage.captureViewport(tab.page, tab.cdp, {
14781483
format: "jpeg",

‎apps/server/src/preview/ServerBrowserPage.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// @effect-diagnostics nodeBuiltinImport:off globalDate:off globalTimers:off - Playwright callbacks run outside the Effect runtime.
22

33
import {
4+
type BrowserRecordingFrameRate,
45
PROVIDER_SEND_TURN_MAX_FILE_BYTES,
56
type PreviewAutomationClickInput,
67
type PreviewAutomationConsoleEntry,
@@ -922,7 +923,7 @@ export const parseNetError = (errorText: string) => {
922923
return { description, code: NET_ERROR_CODES[description] ?? -2 };
923924
};
924925

925-
export const RECORDING_ENCODER_SCRIPT = `(() => {
926+
export const recordingEncoderScript = (frameRate: BrowserRecordingFrameRate) => `(() => {
926927
const canvas = document.createElement("canvas");
927928
const context = canvas.getContext("2d");
928929
const chunks = [];
@@ -991,8 +992,8 @@ export const RECORDING_ENCODER_SCRIPT = `(() => {
991992
canvas.width = bitmap.width;
992993
canvas.height = bitmap.height;
993994
const mimeType = ["video/mp4;codecs=avc1.640033", "video/webm;codecs=vp9", "video/webm"].find((type) => MediaRecorder.isTypeSupported(type));
994-
const bitsPerSecond = Math.min(50e6, Math.max(2.5e6, bitmap.width * bitmap.height * 30 * 0.05));
995-
recorder = new MediaRecorder(canvas.captureStream(30), { mimeType, videoBitsPerSecond: bitsPerSecond });
995+
const bitsPerSecond = Math.min(50e6, Math.max(2.5e6, bitmap.width * bitmap.height * ${frameRate} * 0.05));
996+
recorder = new MediaRecorder(canvas.captureStream(${frameRate}), { mimeType, videoBitsPerSecond: bitsPerSecond });
996997
started = new Promise((resolve, reject) => {
997998
recorder.onstart = resolve;
998999
recorder.onerror = (event) => reject(event.error);

‎apps/web/src/browser/BrowserProfileReporter.tsx‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
1-
import { AuthPreviewOperateScope, type BrowserProfile } from "@t3tools/contracts";
1+
import {
2+
AuthPreviewOperateScope,
3+
type BrowserProfile,
4+
type BrowserRecordingFrameRate,
5+
} from "@t3tools/contracts";
26
import { useEffect } from "react";
37

48
import { useClientSettings, useClientSettingsHydrated } from "~/hooks/useSettings";
@@ -12,11 +16,15 @@ const selectProfiles = (settings: { readonly browserProfiles: ReadonlyArray<Brow
1216
settings.browserProfiles;
1317
const selectDefaultProfileId = (settings: { readonly browserDefaultProfileId: string }) =>
1418
settings.browserDefaultProfileId;
19+
const selectRecordingFrameRate = (settings: {
20+
readonly browserRecordingFrameRate: BrowserRecordingFrameRate;
21+
}) => settings.browserRecordingFrameRate;
1522

1623
export function BrowserProfileReporter() {
1724
const hydrated = useClientSettingsHydrated();
1825
const profiles = useClientSettings(selectProfiles);
1926
const defaultProfileId = useClientSettings(selectDefaultProfileId);
27+
const recordingFrameRate = useClientSettings(selectRecordingFrameRate);
2028
const { environments } = useEnvironments();
2129
const serverConfigs = useServerConfigs();
2230
const operable = useEnvironmentsWithScope(environments, AuthPreviewOperateScope);
@@ -26,9 +34,9 @@ export function BrowserProfileReporter() {
2634
if (!hydrated) return;
2735
for (const [environmentId, config] of serverConfigs) {
2836
if (!config.environment.capabilities.serverBrowser || !operable.has(environmentId)) continue;
29-
void report({ environmentId, input: { profiles, defaultProfileId } });
37+
void report({ environmentId, input: { profiles, defaultProfileId, recordingFrameRate } });
3038
}
31-
}, [defaultProfileId, hydrated, operable, profiles, report, serverConfigs]);
39+
}, [defaultProfileId, hydrated, operable, profiles, recordingFrameRate, report, serverConfigs]);
3240

3341
return null;
3442
}

‎packages/contracts/src/preview.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,9 +241,16 @@ export const PreviewCloseInput = Schema.Struct({
241241
});
242242
export type PreviewCloseInput = typeof PreviewCloseInput.Type;
243243

244+
export const BROWSER_RECORDING_FRAME_RATES = [30, 60] as const;
245+
export const BrowserRecordingFrameRate = Schema.Literals(BROWSER_RECORDING_FRAME_RATES);
246+
export type BrowserRecordingFrameRate = typeof BrowserRecordingFrameRate.Type;
247+
export const DEFAULT_BROWSER_RECORDING_FRAME_RATE: BrowserRecordingFrameRate = 30;
248+
244249
export const PreviewReportProfilesInput = Schema.Struct({
245250
profiles: Schema.Array(BrowserProfile).check(Schema.isMaxLength(BROWSER_PROFILE_MAX_COUNT)),
246251
defaultProfileId: BrowserProfileId,
252+
/** Optional so older clients keep reporting; omitting it keeps the last reported rate. */
253+
recordingFrameRate: Schema.optional(BrowserRecordingFrameRate),
247254
});
248255
export type PreviewReportProfilesInput = typeof PreviewReportProfilesInput.Type;
249256

‎packages/contracts/src/settings.ts‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ import { ProjectScript, type ProjectMutation } from "./project.ts";
3333
import { DEFAULT_RUNTIME_MODE, RuntimeMode } from "./providerPolicy.ts";
3434
import { BrowserProfile, BrowserProfileId, DEFAULT_BROWSER_PROFILE_ID } from "./browserProfile.ts";
3535
import {
36+
BrowserRecordingFrameRate,
37+
DEFAULT_BROWSER_RECORDING_FRAME_RATE,
3638
DEFAULT_PREVIEW_APPEARANCE,
3739
DEFAULT_PREVIEW_ZOOM_FACTOR,
3840
FILL_PREVIEW_VIEWPORT,
@@ -257,10 +259,11 @@ export type DefaultThemePreference = typeof DefaultThemePreference.Type;
257259

258260
export const DEFAULT_BROWSER_VIEWPORT: PreviewViewportSetting = FILL_PREVIEW_VIEWPORT;
259261
export const DEFAULT_BROWSER_AUTO_SHOW_FLOATING_PREVIEW = true;
260-
export const BROWSER_RECORDING_FRAME_RATES = [30, 60] as const;
261-
export const BrowserRecordingFrameRate = Schema.Literals(BROWSER_RECORDING_FRAME_RATES);
262-
export type BrowserRecordingFrameRate = typeof BrowserRecordingFrameRate.Type;
263-
export const DEFAULT_BROWSER_RECORDING_FRAME_RATE: BrowserRecordingFrameRate = 30;
262+
export {
263+
BROWSER_RECORDING_FRAME_RATES,
264+
BrowserRecordingFrameRate,
265+
DEFAULT_BROWSER_RECORDING_FRAME_RATE,
266+
} from "./preview.ts";
264267
export const BrowserLinkTarget = Schema.Literals(["system", "app"]);
265268
export type BrowserLinkTarget = typeof BrowserLinkTarget.Type;
266269
export const DEFAULT_BROWSER_LINK_TARGET: BrowserLinkTarget = "system";

0 commit comments

Comments
 (0)