Skip to content

Commit d5ecb01

Browse files
vanceingallsclaude
andauthored
perf(engine,producer): batch N drawElement frames per CDP round-trip (HF_DE_BATCH) (#1928)
Amortizes per-frame CDP protocol overhead (~3.5-9ms/frame) by looping seek -> paint-wait -> drawElementImage -> createImageBitmap in ONE page.evaluate for runs of consecutive frames; bitmaps still post to the encode worker per frame. Validated on 19 stratified DE comps: median 1.20x on top of worker-encode (to 1.56x), zero damaged frames, edge comps (static-dedup-heavy, clip-cut) bit-identical; mid-batch failure re-captures via the per-frame path (screenshot-fallback semantics preserved). Off by default; opt in with HF_DE_BATCH=4. Co-authored-by: Claude Opus 4.8 <[email protected]>
1 parent 992a9b6 commit d5ecb01

4 files changed

Lines changed: 360 additions & 0 deletions

File tree

‎packages/engine/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ export {
8383
captureFrame,
8484
captureFrameToBuffer,
8585
captureFrameToBufferPipelined,
86+
captureFramesBatchPipelined,
8687
writeCapturedFrame,
8788
discardWarmupCapture,
8889
getCompositionDuration,

‎packages/engine/src/services/drawElementService.ts‎

Lines changed: 237 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -832,3 +832,240 @@ export async function produceDrawElementFrame(
832832

833833
return { encodeResult };
834834
}
835+
836+
/**
837+
* P6 prototype (HF_DE_BATCH): batch-produce N consecutive frames in ONE CDP
838+
* round-trip. In-page loop per frame: `__hf.seek(t)` → paint-wait (tick toggle +
839+
* canvas `paint` event) → drawElementImage composite → createImageBitmap →
840+
* postMessage to the encode worker. Bitmaps are posted per-frame (encode starts
841+
* immediately); only the CDP protocol round-trips are amortized N-fold.
842+
* Micro-pipeline inside the batch: frame i+1's seek/paint-wait overlaps frame
843+
* i's createImageBitmap (the canvas is only redrawn after i's bitmap resolves).
844+
*
845+
* macOS-GPU sync path only (the worker-encode gate guarantees this at the call
846+
* site). On an in-page failure at frame k, frames < k are already at the worker
847+
* (their promises resolve normally); pending entries for frames >= k are
848+
* rejected here and `failedAt` tells the caller to re-capture k.. via the
849+
* per-frame path (which owns the screenshot-fallback semantics).
850+
*/
851+
export async function produceDrawElementFrameBatch(
852+
page: Page,
853+
times: number[],
854+
width: number,
855+
height: number,
856+
quality = 80,
857+
): Promise<{ encodeResults: Array<Promise<Buffer>>; failedAt: number | null; error?: string }> {
858+
const state = workerEncodeStates.get(page);
859+
if (!state) {
860+
throw new Error(
861+
"drawElement worker encode not initialized; call initDrawElementWorkerEncode first",
862+
);
863+
}
864+
865+
const fids: number[] = [];
866+
const encodeResults: Array<Promise<Buffer>> = [];
867+
for (let i = 0; i < times.length; i++) {
868+
const frameId = ++state.nextId;
869+
fids.push(frameId);
870+
const p = new Promise<Buffer>((resolve, reject) => {
871+
const timer = setTimeout(() => {
872+
if (state.pending.delete(frameId)) {
873+
reject(new Error(`drawElement worker encode timed out (frame ${frameId})`));
874+
}
875+
}, 30_000);
876+
state.pending.set(frameId, {
877+
resolve: (b) => {
878+
clearTimeout(timer);
879+
resolve(b);
880+
},
881+
reject: (e) => {
882+
clearTimeout(timer);
883+
reject(e);
884+
},
885+
});
886+
});
887+
void p.catch(() => {}); // same orphan-rejection guard as produceDrawElementFrame
888+
encodeResults.push(p);
889+
}
890+
891+
const outcome = await page.evaluate(
892+
async ({
893+
frames,
894+
w,
895+
h,
896+
q,
897+
}: {
898+
frames: Array<{ t: number; fid: number }>;
899+
w: number;
900+
h: number;
901+
q: number;
902+
}): Promise<{ failedAt: number | null; error?: string }> => {
903+
const canvas = document.getElementById("__hf_de_canvas") as HTMLCanvasElement | null;
904+
const root = document.querySelector("[data-composition-id]") as HTMLElement | null;
905+
if (!canvas || !root) return { failedAt: 0, error: "drawElement canvas not initialized" };
906+
const ctx = canvas.getContext("2d");
907+
if (!ctx) return { failedAt: 0, error: "drawElement: 2d context unavailable" };
908+
909+
type AccelWindow = Window & {
910+
__hf_accel_canvases?: HTMLCanvasElement[];
911+
__hf3d?: { update: () => void };
912+
__hf?: { seek?: (t: number) => void };
913+
__HF_ROOT_PROPS__?: boolean;
914+
__HF_ROOT_BASE_OPACITY__?: number;
915+
__hfEncWorker?: Worker;
916+
};
917+
const aw = window as AccelWindow;
918+
919+
const waitPaint = (): Promise<void> =>
920+
new Promise((res) => {
921+
let done = false;
922+
const settle = () => {
923+
if (done) return;
924+
done = true;
925+
canvas.removeEventListener("paint", settle);
926+
res();
927+
};
928+
canvas.addEventListener("paint", settle);
929+
const tick = document.getElementById("__hf_de_tick");
930+
if (tick) {
931+
tick.style.backgroundColor =
932+
tick.style.backgroundColor === "rgb(0, 0, 0)" ? "rgb(1, 1, 1)" : "rgb(0, 0, 0)";
933+
}
934+
setTimeout(settle, 250);
935+
});
936+
937+
let prevBitmap: Promise<void> = Promise.resolve();
938+
let prevBitmapIdx = -1;
939+
const errMsg = (e: unknown) => (e instanceof Error ? e.message : String(e));
940+
941+
for (let i = 0; i < frames.length; i++) {
942+
const frame = frames[i];
943+
if (!frame) return { failedAt: i, error: "batch frame missing" };
944+
const { t, fid } = frame;
945+
try {
946+
if (aw.__hf && typeof aw.__hf.seek === "function") aw.__hf.seek(t);
947+
aw.__hf3d?.update();
948+
const accel = (aw.__hf_accel_canvases ?? []).filter((c) => root.contains(c));
949+
for (const c of accel) {
950+
if (c.style.visibility !== "hidden") c.style.visibility = "hidden";
951+
}
952+
await waitPaint();
953+
// Wait for the previous frame's bitmap before overwriting the canvas.
954+
try {
955+
await prevBitmap;
956+
} catch (e) {
957+
return { failedAt: prevBitmapIdx, error: errMsg(e) };
958+
}
959+
960+
ctx.clearRect(0, 0, w, h);
961+
let bg = "";
962+
for (let el = root.parentElement; el; el = el.parentElement) {
963+
const c = getComputedStyle(el).backgroundColor;
964+
if (c && c !== "transparent" && c !== "rgba(0, 0, 0, 0)") {
965+
bg = c;
966+
break;
967+
}
968+
}
969+
ctx.fillStyle = bg || "#fff";
970+
ctx.fillRect(0, 0, w, h);
971+
const rootRect = root.getBoundingClientRect();
972+
for (const c of accel) {
973+
if (c.hasAttribute("data-hf-3d")) continue;
974+
const r = c.getBoundingClientRect();
975+
try {
976+
ctx.drawImage(c, r.left - rootRect.left, r.top - rootRect.top, r.width, r.height);
977+
} catch {
978+
// skip
979+
}
980+
}
981+
// Root compositor-applied opacity/transform correction — mirrors
982+
// produceDrawElementFrame (see its comment).
983+
let appliedAlpha = false;
984+
let appliedTransform = false;
985+
if (aw.__HF_ROOT_PROPS__) {
986+
try {
987+
const rcs = getComputedStyle(root);
988+
const baseOp = aw.__HF_ROOT_BASE_OPACITY__ ?? 1;
989+
const curOp = parseFloat(rcs.opacity);
990+
if (baseOp > 0.001 && Number.isFinite(curOp)) {
991+
const ratio = curOp / baseOp;
992+
if (Math.abs(ratio - 1) > 0.002) {
993+
ctx.globalAlpha = Math.max(0, Math.min(1, ratio));
994+
appliedAlpha = true;
995+
}
996+
}
997+
const curTransform = rcs.transform;
998+
if (curTransform && curTransform !== "none") {
999+
const m = new DOMMatrix(curTransform);
1000+
const origin = rcs.transformOrigin.split(" ");
1001+
const ox = parseFloat(origin[0] ?? "0") || 0;
1002+
const oy = parseFloat(origin[1] ?? "0") || 0;
1003+
ctx.translate(ox, oy);
1004+
ctx.transform(m.a, m.b, m.c, m.d, m.e, m.f);
1005+
ctx.translate(-ox, -oy);
1006+
appliedTransform = true;
1007+
}
1008+
} catch {
1009+
/* leave context unchanged → uncorrected (no worse than before) */
1010+
}
1011+
}
1012+
(
1013+
ctx as unknown as { drawElementImage(el: Element, x: number, y: number): void }
1014+
).drawElementImage(root, 0, 0);
1015+
if (appliedAlpha) ctx.globalAlpha = 1;
1016+
if (appliedTransform) ctx.setTransform(1, 0, 0, 1, 0, 0);
1017+
for (const c of accel) {
1018+
if (!c.hasAttribute("data-hf-3d")) continue;
1019+
const r = c.getBoundingClientRect();
1020+
try {
1021+
ctx.drawImage(c, r.left - rootRect.left, r.top - rootRect.top, r.width, r.height);
1022+
} catch {
1023+
// skip
1024+
}
1025+
}
1026+
1027+
prevBitmapIdx = i;
1028+
prevBitmap = createImageBitmap(canvas).then((bmp) => {
1029+
if (!aw.__hfEncWorker) {
1030+
bmp.close();
1031+
throw new Error("drawElement: encode worker not initialized");
1032+
}
1033+
aw.__hfEncWorker.postMessage({ bmp, id: fid, w, h, q: q / 100 }, [bmp]);
1034+
});
1035+
} catch (e) {
1036+
try {
1037+
await prevBitmap;
1038+
} catch {
1039+
/* prior frame's failure surfaces via its own pending timeout path */
1040+
}
1041+
return { failedAt: i, error: errMsg(e) };
1042+
}
1043+
}
1044+
try {
1045+
await prevBitmap;
1046+
} catch (e) {
1047+
return { failedAt: prevBitmapIdx, error: errMsg(e) };
1048+
}
1049+
return { failedAt: null };
1050+
},
1051+
{ frames: times.map((t, i) => ({ t, fid: fids[i] ?? 0 })), w: width, h: height, q: quality },
1052+
);
1053+
1054+
if (outcome.failedAt !== null) {
1055+
// Frames >= failedAt never reached the worker — reject their pendings now
1056+
// so nothing waits 30s on the watchdog.
1057+
for (let k = outcome.failedAt; k < fids.length; k++) {
1058+
const fid = fids[k];
1059+
if (fid === undefined) continue;
1060+
const entry = state.pending.get(fid);
1061+
if (entry) {
1062+
state.pending.delete(fid);
1063+
entry.reject(
1064+
new Error(`drawElement batch produce failed at frame ${k}: ${outcome.error ?? "?"}`),
1065+
);
1066+
}
1067+
}
1068+
}
1069+
1070+
return { encodeResults, failedAt: outcome.failedAt, error: outcome.error };
1071+
}

‎packages/engine/src/services/frameCapture.ts‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import {
3939
initDrawElementWorkerEncode,
4040
cleanupDrawElementWorkerEncode,
4141
produceDrawElementFrame,
42+
produceDrawElementFrameBatch,
4243
} from "./drawElementService.js";
4344
import { initThreeDProjection, detectCssEffectRisk } from "./threeDProjection.js";
4445
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
@@ -2511,6 +2512,73 @@ export async function captureFrameToBufferPipelined(
25112512
}
25122513
}
25132514

2515+
/**
2516+
* P6 prototype (HF_DE_BATCH): capture N consecutive frames in one CDP
2517+
* round-trip via {@link produceDrawElementFrameBatch}. The caller pre-plans the
2518+
* batch (consecutive frame indices, none static-dedup'd, none opt-in
2519+
* boundary-screenshot). On a mid-batch in-page failure the remaining frames are
2520+
* re-captured through {@link captureFrameToBufferPipelined}, which owns the
2521+
* per-frame screenshot-fallback semantics — so failure behavior is identical to
2522+
* the unbatched path, just discovered at batch granularity.
2523+
*/
2524+
export async function captureFramesBatchPipelined(
2525+
session: CaptureSession,
2526+
frameIndices: number[],
2527+
times: number[],
2528+
): Promise<Array<{ frameIndex: number; encodeResult: Promise<Buffer> }>> {
2529+
const { page, options } = session;
2530+
if (!session.isInitialized) {
2531+
throw new Error("[FrameCapture] Session not initialized");
2532+
}
2533+
const startTime = Date.now();
2534+
const fps = fpsToNumber(options.fps);
2535+
const quantized = times.map((t) => quantizeTimeToFrame(t, fps));
2536+
2537+
const { encodeResults, failedAt, error } = await produceDrawElementFrameBatch(
2538+
page,
2539+
quantized,
2540+
options.width,
2541+
options.height,
2542+
options.quality ?? 80,
2543+
);
2544+
2545+
const okCount = failedAt === null ? frameIndices.length : failedAt;
2546+
const elapsed = Date.now() - startTime;
2547+
session.capturePerf.frames += okCount;
2548+
// Round-trips are fused — attribute the whole batch to produce time.
2549+
session.capturePerf.screenshotMs += elapsed;
2550+
session.capturePerf.totalMs += elapsed;
2551+
2552+
const results: Array<{ frameIndex: number; encodeResult: Promise<Buffer> }> = [];
2553+
for (let i = 0; i < okCount; i++) {
2554+
const frameIndex = frameIndices[i];
2555+
const encodeResult = encodeResults[i];
2556+
if (frameIndex === undefined || !encodeResult) break;
2557+
results.push({ frameIndex, encodeResult });
2558+
}
2559+
2560+
if (failedAt !== null) {
2561+
console.log(
2562+
`[engine] fast capture: batch produce failed at frame ` +
2563+
`${frameIndices[failedAt] ?? "?"} (${error ?? "?"}); ` +
2564+
`re-capturing ${frameIndices.length - failedAt} frame(s) per-frame`,
2565+
);
2566+
for (let i = failedAt; i < frameIndices.length; i++) {
2567+
const frameIndex = frameIndices[i];
2568+
const time = times[i];
2569+
if (frameIndex === undefined || time === undefined) break;
2570+
const { encodeResult } = await captureFrameToBufferPipelined(session, frameIndex, time);
2571+
results.push({ frameIndex, encodeResult });
2572+
}
2573+
}
2574+
2575+
// Task B: retain the last encode result so a following static frame can reuse it.
2576+
const last = results[results.length - 1];
2577+
if (session.staticFrames && last) session.lastEncodeResult = last.encodeResult;
2578+
2579+
return results;
2580+
}
2581+
25142582
/**
25152583
* Type of the "inner capture" function consumed by
25162584
* {@link discardWarmupCapture}. Matches the real `captureFrameCore` signature

‎packages/producer/src/services/render/stages/captureStreamingStage.ts‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ import {
5050
type StreamingEncoder,
5151
captureFrameToBuffer,
5252
captureFrameToBufferPipelined,
53+
captureFramesBatchPipelined,
5354
closeCaptureSession,
5455
createCaptureSession,
5556
createFrameReorderBuffer,
@@ -167,6 +168,59 @@ async function runWorkerEncodePipelineLoop(
167168
);
168169
};
169170

171+
// P6 (HF_DE_BATCH=N, N>1): capture runs of consecutive frames in ONE CDP
172+
// round-trip each (in-page seek+paint+draw loop), amortizing protocol latency
173+
// N-fold. Batches break at static-dedup frames (skipped via lastEncodeResult
174+
// reuse — order-dependent) and at opt-in boundary-screenshot frames; those go
175+
// through the per-frame path unchanged. onBeforeCapture (video frame
176+
// injection) needs a node-side hook per frame → no batching.
177+
const batchN = Math.floor(Number(process.env.HF_DE_BATCH ?? "0"));
178+
if (batchN > 1 && !session.onBeforeCapture) {
179+
const frameTime = (f: number) => (f * job.config.fps.den) / job.config.fps.num;
180+
const drainBatch = async (batch: Array<{ idx: number; encodeResult: Promise<Buffer> }>) => {
181+
for (const item of batch) {
182+
assertNotAborted();
183+
const buf = await item.encodeResult;
184+
await reorderBuffer.waitForFrame(item.idx);
185+
ensureFrameWritten(await currentEncoder.writeFrame(buf), item.idx, currentEncoder);
186+
reorderBuffer.advanceTo(item.idx + 1);
187+
job.framesRendered = item.idx + 1;
188+
updateJobStatus(
189+
job,
190+
"rendering",
191+
`Streaming frame ${item.idx + 1}/${totalFrames}`,
192+
Math.round(25 + ((item.idx + 1) / totalFrames) * 55),
193+
onProgress,
194+
);
195+
}
196+
};
197+
const boundarySS = process.env.HF_FAST_CAPTURE_BOUNDARY_SS === "true";
198+
const batchable = (f: number) =>
199+
!session.staticFrames?.has(f) && !(boundarySS && session.clipBoundaryFrames?.has(f));
200+
let prevBatch: Array<{ idx: number; encodeResult: Promise<Buffer> }> = [];
201+
let i = 0;
202+
while (i < totalFrames) {
203+
assertNotAborted();
204+
if (batchable(i)) {
205+
const idxs: number[] = [];
206+
while (idxs.length < batchN && i < totalFrames && batchable(i)) {
207+
idxs.push(i);
208+
i++;
209+
}
210+
const results = await captureFramesBatchPipelined(session, idxs, idxs.map(frameTime));
211+
await drainBatch(prevBatch);
212+
prevBatch = results.map((r) => ({ idx: r.frameIndex, encodeResult: r.encodeResult }));
213+
} else {
214+
const { encodeResult } = await captureFrameToBufferPipelined(session, i, frameTime(i));
215+
await drainBatch(prevBatch);
216+
prevBatch = [{ idx: i, encodeResult }];
217+
i++;
218+
}
219+
}
220+
await drainBatch(prevBatch);
221+
return;
222+
}
223+
170224
// On abort/throw the just-produced frame's encode is still in flight and never
171225
// awaited (it isn't `prev` yet); cleanupDrawElementWorkerEncode rejects it on
172226
// close. produceDrawElementFrame attaches a no-op catch to every encodeResult

0 commit comments

Comments
 (0)