Skip to content

Commit 9852036

Browse files
feat(producer): opt-in video layer passthrough for a lone full-canvas video (#6914)
* feat(engine): composite a source video inside the streaming encoder * feat(engine): opt-in flag and rule for compositing a lone full-timeline video * feat(producer): opt-in video layer passthrough for a lone full-canvas video * feat(producer): launch the probe, drop its page and capture PNG when passthrough is on * refactor(engine): trim the passthrough doc comments to one line * fix(producer): refuse passthrough when a wrapper or sub-composition paints a background * chore(producer): mark the in-page layout check as one deliberate unit * fix(engine): a video left out of capture no longer blocks static-frame dedup * refactor(engine): shorten two long comments in the static-frame eligibility scan
1 parent a7075fc commit 9852036

15 files changed

Lines changed: 620 additions & 136 deletions

‎packages/engine/src/config.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ export interface EngineConfig {
7070
* `HF_STATIC_DEDUP` in {false,0,off}. Only arms in screenshot capture mode.
7171
*/
7272
staticFrameDedup: boolean;
73+
/** Opt-in `HF_VIDEO_LAYER_PASSTHROUGH`: ffmpeg composites a full-canvas source video under the page capture. */
74+
videoLayerPassthrough: boolean;
7375
/**
7476
* Use drawElementImage for frame capture (requires the CanvasDrawElement
7577
* Chrome flag, added globally in buildChromeArgs). Default ON, clamped in
@@ -301,6 +303,7 @@ export const DEFAULT_CONFIG: EngineConfig = {
301303
protocolTimeout: 300_000,
302304
forceScreenshot: false,
303305
staticFrameDedup: true,
306+
videoLayerPassthrough: false,
304307
useDrawElement: true,
305308
enableDrawElementWorkerEncode: true,
306309
// Auto-detected per host in `resolveConfig`; defaults off for the raw
@@ -354,6 +357,7 @@ const BOOLEAN_ENGINE_CONFIG_FIELDS = [
354357
"enableBrowserPool",
355358
"forceScreenshot",
356359
"staticFrameDedup",
360+
"videoLayerPassthrough",
357361
"useDrawElement",
358362
"enableDrawElementWorkerEncode",
359363
"lowMemoryMode",
@@ -876,6 +880,10 @@ export function resolveConfig(overrides?: Partial<EngineConfig>): EngineConfig {
876880

877881
forceScreenshot: envBool("PRODUCER_FORCE_SCREENSHOT", DEFAULT_CONFIG.forceScreenshot),
878882
staticFrameDedup: resolveStaticFrameDedup(),
883+
videoLayerPassthrough: envBool(
884+
"HF_VIDEO_LAYER_PASSTHROUGH",
885+
DEFAULT_CONFIG.videoLayerPassthrough,
886+
),
879887
useDrawElement: envBool("PRODUCER_EXPERIMENTAL_FAST_CAPTURE", DEFAULT_CONFIG.useDrawElement),
880888
enableDrawElementWorkerEncode: envBool(
881889
"HF_DE_WORKER_ENCODE",

‎packages/engine/src/index.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,10 @@ export {
267267
isVideoFrameFormat,
268268
EXTRACT_CACHE_MIN_AGE_MS,
269269
sdrCanvasColourFilters,
270+
buildVideoLayerInput,
271+
type VideoLayerInput,
270272
} from "./services/videoFrameExtractor.js";
273+
export { qualifyVideoLayer } from "./services/videoLayerPassthrough.js";
271274
export { isSelfOrAncestorHidden } from "./services/mediaHidden.js";
272275
export { directorySizeBytes, gcExtractionCache } from "./services/extractionCache.js";
273276

‎packages/engine/src/services/frameCapture.ts‎

Lines changed: 123 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -1265,7 +1265,11 @@ async function initDrawElementOrTransparentBackground(
12651265
await finalizeDrawElementInit(session, page, logInitPhase, { transparent, forceDE });
12661266
}
12671267
} else if (capturesAlpha(session.options)) {
1268-
await initTransparentBackground(session.page);
1268+
const hideElementIds = session.options.hiddenElementIds ?? [];
1269+
await initTransparentBackground(session.page, {
1270+
clearCompositionRoot: hideElementIds.length > 0,
1271+
hideElementIds,
1272+
});
12691273
}
12701274
}
12711275

@@ -3416,6 +3420,7 @@ export function isStaticDedupFrameAnalysisSafe(totalFrames: number): boolean {
34163420
export async function computeStaticFrameSet(
34173421
page: Page,
34183422
fps: number,
3423+
ignoredVideoIds: readonly string[] = [],
34193424
): Promise<{
34203425
totalFrames: number;
34213426
staticFrameSet: Set<number>;
@@ -3429,131 +3434,121 @@ export async function computeStaticFrameSet(
34293434
eligible: boolean;
34303435
reason: string;
34313436
}> {
3432-
const result = await page.evaluate((spatialProps: readonly string[]) => {
3433-
type AnyTween = {
3434-
startTime(): number;
3435-
duration(): number;
3436-
totalDuration?(): number;
3437-
getChildren?(nested: boolean, tweens: boolean, timelines: boolean): AnyTween[];
3438-
vars?: Record<string, unknown>;
3439-
};
3440-
// Passed in from SPATIAL_TWEEN_PROPERTIES rather than declared here, so the list
3441-
// stays the one this module's own tests exercise (the matching code itself must
3442-
// stay inline — this closure is serialized and runs in the page realm).
3443-
const SPATIAL_PROPS = new Set(spatialProps);
3444-
function isSpatial(vars: Record<string, unknown> | undefined): boolean {
3445-
if (!vars) return false;
3446-
for (const key of Object.keys(vars)) if (SPATIAL_PROPS.has(key)) return true;
3447-
return false;
3448-
}
3449-
const intervals: Array<{ start: number; end: number; spatial: boolean }> = [];
3450-
let tweenCount = 0;
3451-
// totalDuration() (NOT duration()): a repeat/yoyo tween animates past one iteration;
3452-
// a repeating timeline is marked opaque over its whole span (conservative).
3453-
// A GSAP tl.call() is a zero-duration tween whose vars wire the callback as
3454-
// onComplete (and onReverseComplete, fired on backward crossing — GSAP has
3455-
// no separate "undo" callback, so both directions invoke the SAME forward
3456-
// side effect). A one-shot DOM mutation driven this way (e.g. a counter's
3457-
// textContent) is not seek-idempotent: crossing it during the static-dedup
3458-
// verifier's own arm-time seeking permanently mutates the page, and that
3459-
// corruption can leak into a LATER, unrelated static run's real capture
3460-
// (the verifier's mismatch check only catches drift within the run being
3461-
// checked, not contamination from a run checked afterward). No reliable
3462-
// way to tell a DOM-mutating call() from a harmless one (analytics ping,
3463-
// class toggle with no visual effect) without executing it, so disqualify
3464-
// the whole comp on ANY call() rather than risk shipping wrong pixels.
3465-
let hasTimelineCall = false;
3466-
function walk(tl: AnyTween, offset: number): void {
3467-
if (typeof tl.getChildren !== "function") return;
3468-
// A timeline with an onUpdate callback can drive motion without any
3469-
// visible tween (e.g. setting x/y from Math.random inside onUpdate).
3470-
// Mark its entire span as animated so those frames are never deduped.
3471-
if (typeof tl.vars?.onUpdate === "function") {
3472-
const total = typeof tl.totalDuration === "function" ? tl.totalDuration() : 0;
3473-
// onUpdate can move anything (e.g. x/y from Math.random) — conservatively spatial.
3474-
if (total > 0) intervals.push({ start: offset, end: offset + total, spatial: true });
3437+
const result = await page.evaluate(
3438+
(spatialProps: readonly string[], ignored: string[]) => {
3439+
type AnyTween = {
3440+
startTime(): number;
3441+
duration(): number;
3442+
totalDuration?(): number;
3443+
getChildren?(nested: boolean, tweens: boolean, timelines: boolean): AnyTween[];
3444+
vars?: Record<string, unknown>;
3445+
};
3446+
// Passed in from SPATIAL_TWEEN_PROPERTIES rather than declared here, so the list
3447+
// stays the one this module's own tests exercise (the matching code itself must
3448+
// stay inline — this closure is serialized and runs in the page realm).
3449+
const SPATIAL_PROPS = new Set(spatialProps);
3450+
function isSpatial(vars: Record<string, unknown> | undefined): boolean {
3451+
if (!vars) return false;
3452+
for (const key of Object.keys(vars)) if (SPATIAL_PROPS.has(key)) return true;
3453+
return false;
34753454
}
3476-
for (const child of tl.getChildren(false, true, true)) {
3477-
const start = offset + (typeof child.startTime === "function" ? child.startTime() : 0);
3478-
const single = typeof child.duration === "function" ? child.duration() : 0;
3479-
const total = typeof child.totalDuration === "function" ? child.totalDuration() : single;
3480-
if (typeof child.getChildren === "function") {
3481-
if (total > single + 1e-6) {
3482-
// A repeating nested timeline's own children are walked for their real
3483-
// vars below; this span is a conservative (spatial) placeholder for the
3484-
// repeat/yoyo overrun, which a per-child walk doesn't otherwise cover.
3485-
intervals.push({ start, end: start + total, spatial: true });
3486-
// Still descend for hasTimelineCall even though the repeating
3487-
// span is already opaque (its frames are excluded from dedup
3488-
// regardless): a call() inside it is a review-flagged detection
3489-
// gap otherwise — the arm-time verifier can still forward-seek
3490-
// through this span while checking a LATER static run, firing
3491-
// the call() and corrupting the page (review).
3492-
walk(child, start);
3455+
const intervals: Array<{ start: number; end: number; spatial: boolean }> = [];
3456+
let tweenCount = 0;
3457+
// totalDuration(), not duration(): repeat/yoyo tweens animate past one iteration.
3458+
// Any tl.call() disqualifies the comp: its DOM side effects are not seek-idempotent.
3459+
let hasTimelineCall = false;
3460+
function walk(tl: AnyTween, offset: number): void {
3461+
if (typeof tl.getChildren !== "function") return;
3462+
// A timeline with an onUpdate callback can drive motion without any
3463+
// visible tween (e.g. setting x/y from Math.random inside onUpdate).
3464+
// Mark its entire span as animated so those frames are never deduped.
3465+
if (typeof tl.vars?.onUpdate === "function") {
3466+
const total = typeof tl.totalDuration === "function" ? tl.totalDuration() : 0;
3467+
// onUpdate can move anything (e.g. x/y from Math.random) — conservatively spatial.
3468+
if (total > 0) intervals.push({ start: offset, end: offset + total, spatial: true });
3469+
}
3470+
for (const child of tl.getChildren(false, true, true)) {
3471+
const start = offset + (typeof child.startTime === "function" ? child.startTime() : 0);
3472+
const single = typeof child.duration === "function" ? child.duration() : 0;
3473+
const total = typeof child.totalDuration === "function" ? child.totalDuration() : single;
3474+
if (typeof child.getChildren === "function") {
3475+
if (total > single + 1e-6) {
3476+
// A repeating nested timeline's own children are walked for their real
3477+
// vars below; this span is a conservative (spatial) placeholder for the
3478+
// repeat/yoyo overrun, which a per-child walk doesn't otherwise cover.
3479+
intervals.push({ start, end: start + total, spatial: true });
3480+
// Still descend: a call() inside the repeating span would otherwise go undetected.
3481+
walk(child, start);
3482+
} else {
3483+
walk(child, start);
3484+
}
34933485
} else {
3494-
walk(child, start);
3495-
}
3496-
} else {
3497-
tweenCount++;
3498-
intervals.push({ start, end: start + total, spatial: isSpatial(child.vars) });
3499-
if (
3500-
total <= 1e-6 &&
3501-
(typeof child.vars?.onComplete === "function" ||
3502-
typeof child.vars?.onReverseComplete === "function")
3503-
) {
3504-
hasTimelineCall = true;
3486+
tweenCount++;
3487+
intervals.push({ start, end: start + total, spatial: isSpatial(child.vars) });
3488+
if (
3489+
total <= 1e-6 &&
3490+
(typeof child.vars?.onComplete === "function" ||
3491+
typeof child.vars?.onReverseComplete === "function")
3492+
) {
3493+
hasTimelineCall = true;
3494+
}
35053495
}
35063496
}
35073497
}
3508-
}
3509-
const w = window as unknown as {
3510-
__timelines?: Record<string, AnyTween>;
3511-
__hf?: { duration?: number };
3512-
__hfHasFrameSources?: () => boolean;
3513-
};
3514-
for (const tl of Object.values(w.__timelines || {})) {
3515-
if (tl && typeof tl.getChildren === "function") walk(tl, 0);
3516-
}
3517-
const hasVideo = !!document.querySelector("video");
3518-
const hasCanvas = !!document.querySelector("canvas");
3519-
const hasFrameSources = w.__hfHasFrameSources?.() ?? false;
3520-
const hasIframe = !!document.querySelector("iframe");
3521-
// A non-numeric data-start (reference expression like "intro+0.5") can't be turned
3522-
// into a clip-cut boundary by computeClipBoundaryFrames' parseFloat, so the cut goes
3523-
// unprotected and could be deduped into the previous scene. Disqualify the comp.
3524-
const hasUnresolvableClipStart = Array.from(document.querySelectorAll("[data-start]")).some(
3525-
(el) => {
3526-
const v = (el as HTMLElement).dataset.start;
3527-
return v != null && v.trim() !== "" && !Number.isFinite(parseFloat(v));
3528-
},
3529-
);
3530-
// Non-GSAP animation (CSS @keyframes / transitions / WAAPI) surfaces via
3531-
// getAnimations(); any running/paused one can change content without a tween.
3532-
let hasNonGsapAnim = false;
3533-
try {
3534-
const docAnims = (document as unknown as { getAnimations?: () => Animation[] }).getAnimations;
3535-
if (typeof docAnims === "function") {
3536-
hasNonGsapAnim = docAnims.call(document).some((a) => {
3537-
const t = a as Animation & { playState?: string };
3538-
return t.playState === "running" || t.playState === "paused";
3539-
});
3498+
const w = window as unknown as {
3499+
__timelines?: Record<string, AnyTween>;
3500+
__hf?: { duration?: number };
3501+
__hfHasFrameSources?: () => boolean;
3502+
};
3503+
for (const tl of Object.values(w.__timelines || {})) {
3504+
if (tl && typeof tl.getChildren === "function") walk(tl, 0);
35403505
}
3541-
} catch {
3542-
hasNonGsapAnim = true;
3543-
}
3544-
return {
3545-
intervals,
3546-
tweenCount,
3547-
duration: w.__hf?.duration ?? 0,
3548-
hasVideo,
3549-
hasCanvas,
3550-
hasFrameSources,
3551-
hasIframe,
3552-
hasNonGsapAnim,
3553-
hasUnresolvableClipStart,
3554-
hasTimelineCall,
3555-
};
3556-
}, SPATIAL_TWEEN_PROPERTIES);
3506+
const hasVideo = Array.from(document.querySelectorAll("video")).some(
3507+
(v) => !ignored.includes(v.id),
3508+
);
3509+
const hasCanvas = !!document.querySelector("canvas");
3510+
const hasFrameSources = w.__hfHasFrameSources?.() ?? false;
3511+
const hasIframe = !!document.querySelector("iframe");
3512+
// A non-numeric data-start (reference expression like "intro+0.5") can't be turned
3513+
// into a clip-cut boundary by computeClipBoundaryFrames' parseFloat, so the cut goes
3514+
// unprotected and could be deduped into the previous scene. Disqualify the comp.
3515+
const hasUnresolvableClipStart = Array.from(document.querySelectorAll("[data-start]")).some(
3516+
(el) => {
3517+
const v = (el as HTMLElement).dataset.start;
3518+
return v != null && v.trim() !== "" && !Number.isFinite(parseFloat(v));
3519+
},
3520+
);
3521+
// Non-GSAP animation (CSS @keyframes / transitions / WAAPI) surfaces via
3522+
// getAnimations(); any running/paused one can change content without a tween.
3523+
let hasNonGsapAnim = false;
3524+
try {
3525+
const docAnims = (document as unknown as { getAnimations?: () => Animation[] })
3526+
.getAnimations;
3527+
if (typeof docAnims === "function") {
3528+
hasNonGsapAnim = docAnims.call(document).some((a) => {
3529+
const t = a as Animation & { playState?: string };
3530+
return t.playState === "running" || t.playState === "paused";
3531+
});
3532+
}
3533+
} catch {
3534+
hasNonGsapAnim = true;
3535+
}
3536+
return {
3537+
intervals,
3538+
tweenCount,
3539+
duration: w.__hf?.duration ?? 0,
3540+
hasVideo,
3541+
hasCanvas,
3542+
hasFrameSources,
3543+
hasIframe,
3544+
hasNonGsapAnim,
3545+
hasUnresolvableClipStart,
3546+
hasTimelineCall,
3547+
};
3548+
},
3549+
SPATIAL_TWEEN_PROPERTIES,
3550+
[...ignoredVideoIds],
3551+
);
35573552

35583553
const {
35593554
intervals,
@@ -4027,7 +4022,11 @@ async function armStaticDedup(
40274022
session.options.motionBlur.samplesPerFrame === undefined &&
40284023
!session.motionBlurNonSpatialFrames
40294024
) {
4030-
sharedStaticFrameStats = await computeStaticFrameSet(page, fpsToNumber(session.options.fps));
4025+
sharedStaticFrameStats = await computeStaticFrameSet(
4026+
page,
4027+
fpsToNumber(session.options.fps),
4028+
session.options.hiddenElementIds,
4029+
);
40314030
session.motionBlurNonSpatialFrames = sharedStaticFrameStats.nonSpatialOnlyFrameSet;
40324031
}
40334032
// Idempotent: the drawElement init path arms dedup BEFORE canvas injection
@@ -4079,7 +4078,9 @@ async function armStaticDedup(
40794078
return;
40804079
}
40814080
const fps = fpsToNumber(session.options.fps);
4082-
const stats = sharedStaticFrameStats ?? (await computeStaticFrameSet(page, fps));
4081+
const stats =
4082+
sharedStaticFrameStats ??
4083+
(await computeStaticFrameSet(page, fps, session.options.hiddenElementIds));
40834084
if (!stats.eligible || stats.staticFrameSet.size === 0) {
40844085
session.staticDedupSkipReason = "ineligible";
40854086
logInitPhase(`static-frame dedup: disabled (${stats.reason})`);

‎packages/engine/src/services/screenshotService.ts‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -293,25 +293,32 @@ const TRANSPARENT_BG_STYLE_ID = "__hf_transparent_bg__";
293293

294294
export async function initTransparentBackground(
295295
page: Page,
296-
{ clearCompositionRoot = false }: { clearCompositionRoot?: boolean } = {},
296+
{
297+
clearCompositionRoot = false,
298+
hideElementIds = [],
299+
}: { clearCompositionRoot?: boolean; hideElementIds?: readonly string[] } = {},
297300
): Promise<void> {
298301
const client = await getCdpSession(page);
299302
await client.send("Emulation.setDefaultBackgroundColorOverride", {
300303
color: { r: 0, g: 0, b: 0, a: 0 },
301304
});
302305
await page.evaluate(
303-
(styleId: string, clearCompositionRoot: boolean) => {
306+
(styleId: string, clearCompositionRoot: boolean, hideIds: string[]) => {
304307
const selector = clearCompositionRoot ? "html,body,[data-composition-id]" : "html,body";
305308
let style = document.getElementById(styleId);
306309
if (!style) {
307310
style = document.createElement("style");
308311
style.id = styleId;
309312
document.head.appendChild(style);
310313
}
311-
style.textContent = `${selector}{background:transparent !important;background-color:transparent !important;background-image:none !important;}`;
314+
const hidden = hideIds.map((id) => `#${CSS.escape(id)}`).join(",");
315+
style.textContent =
316+
`${selector}{background:transparent !important;background-color:transparent !important;background-image:none !important;}` +
317+
(hidden ? `${hidden}{visibility:hidden !important;}` : "");
312318
},
313319
TRANSPARENT_BG_STYLE_ID,
314320
clearCompositionRoot,
321+
[...hideElementIds],
315322
);
316323
}
317324

‎packages/engine/src/services/streamingEncoder.test.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1396,3 +1396,29 @@ describe.skipIf(spawnSync("ffmpeg", ["-version"]).status !== 0)(
13961396
}, 30_000);
13971397
},
13981398
);
1399+
1400+
describe("buildStreamingArgs video layer", () => {
1401+
const layer = { inputArgs: ["-ss", "0", "-i", "film.mp4", "-t", "10"], filters: ["fps=30"] };
1402+
const withLayer = { ...baseSdr, imageFormat: "png" as const, videoLayer: layer };
1403+
1404+
it("overlays the piped frames on the source video and maps the result", () => {
1405+
const args = buildStreamingArgs(withLayer, "/tmp/out.mp4");
1406+
expect(args.indexOf("-i")).toBeLessThan(args.indexOf("film.mp4"));
1407+
const graph = args[args.indexOf("-filter_complex") + 1] ?? "";
1408+
expect(graph).toMatch(
1409+
/^\[1:v\]fps=30,format=gbrp\[v\];\[0:v\]format=rgba\[o\];\[v\]\[o\]overlay=/,
1410+
);
1411+
expect(args[args.indexOf("-map") + 1]).toBe("[vout]");
1412+
expect(args).not.toContain("-vf");
1413+
});
1414+
1415+
it.each([
1416+
["JPEG frames", { imageFormat: "jpeg" as const }],
1417+
["GPU encode", { useGpu: true }],
1418+
["vp9", { codec: "vp9" as const }],
1419+
])("refuses %s", (_name, over) => {
1420+
expect(() => buildStreamingArgs({ ...withLayer, ...over }, "/tmp/out.mp4", "nvenc")).toThrow(
1421+
/videoLayer needs/,
1422+
);
1423+
});
1424+
});

0 commit comments

Comments
 (0)