Skip to content

Commit 7ea65ab

Browse files
committed
fix(producer): gate drawElement on WebGL compositions to preserve multi-worker screenshot (heygen-com#5604)
Compositions with WebGL or accelerated canvases of >= 900 frames were qualifying for the single-worker DrawElement priority inversion, forcing them onto 1 worker and degrading render performance compared to multi-worker screenshot capture. - Add detectWebGlUsage() in htmlCompiler to statically detect WebGL, WebGPU, Three.js, Babylon, Pixi, Mapbox, and MapLibre - Clear cfg.useDrawElement and set deCompileGate to 'webgl' during compileStage, and restore page-side compositing when auto-disabled - Guard against runtime WebGL adapter usage in renderOrchestrator and cleanly discard probe sessions initialized in drawElement mode - Export isWebGlRouteActive helper and include WebGL guards in layeredOrEffectRoute for DE single-worker and parallel router predicates, respecting HF_FAST_CAPTURE_WEBGL and HF_FAST_CAPTURE_CSSFX bypasses - Add tests in htmlCompiler.test.ts, compileStage.test.ts, and renderOrchestrator.test.ts verifying WebGL compositions >= 900 frames preserve multi-worker screenshot capture
1 parent d0da232 commit 7ea65ab

6 files changed

Lines changed: 284 additions & 6 deletions

File tree

‎packages/producer/src/services/htmlCompiler.test.ts‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import {
1818
detectRenderModeHints,
1919
detectShaderTransitionUsage,
2020
detectThreeDTransformUsage,
21+
detectWebGlUsage,
2122
discoverMediaFromBrowser,
2223
discoverAudioVolumeAutomationFromTimeline,
2324
discoverVideoVisibilityFromTimeline,
@@ -1005,6 +1006,73 @@ describe("detectThreeDTransformUsage", () => {
10051006
});
10061007
});
10071008

1009+
describe("detectWebGlUsage", () => {
1010+
it("detects getContext('webgl') and getContext('webgl2')", () => {
1011+
expect(detectWebGlUsage('<script>const gl = canvas.getContext("webgl");</script>')).toBe(true);
1012+
expect(detectWebGlUsage("<script>const gl = canvas.getContext('webgl2');</script>")).toBe(true);
1013+
expect(
1014+
detectWebGlUsage('<script>const gl = canvas.getContext("experimental-webgl");</script>'),
1015+
).toBe(true);
1016+
});
1017+
1018+
it("detects WebGPU contexts and attributes", () => {
1019+
expect(detectWebGlUsage('<script>const context = canvas.getContext("webgpu");</script>')).toBe(
1020+
true,
1021+
);
1022+
expect(detectWebGlUsage('<div data-requires-webgpu id="render"></div>')).toBe(true);
1023+
});
1024+
1025+
it("detects Three.js references and renderer constructors", () => {
1026+
expect(detectWebGlUsage("<script>const renderer = new THREE.WebGLRenderer();</script>")).toBe(
1027+
true,
1028+
);
1029+
expect(detectWebGlUsage("<script>const scene = new THREE.Scene();</script>")).toBe(true);
1030+
expect(
1031+
detectWebGlUsage(
1032+
"<script>const cam = new THREE.PerspectiveCamera(75, 1, 0.1, 1000);</script>",
1033+
),
1034+
).toBe(true);
1035+
expect(detectWebGlUsage("<script>if (window.THREE) { init(); }</script>")).toBe(true);
1036+
});
1037+
1038+
it("detects Babylon.js and PixiJS", () => {
1039+
expect(
1040+
detectWebGlUsage("<script>const engine = new BABYLON.Engine(canvas, true);</script>"),
1041+
).toBe(true);
1042+
expect(detectWebGlUsage("<script>const app = new PIXI.Application();</script>")).toBe(true);
1043+
});
1044+
1045+
it("detects framework CDN scripts", () => {
1046+
expect(
1047+
detectWebGlUsage(
1048+
'<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>',
1049+
),
1050+
).toBe(true);
1051+
expect(detectWebGlUsage('<script src="https://cdn.example.com/pixi.js"></script>')).toBe(true);
1052+
expect(
1053+
detectWebGlUsage(
1054+
'<script src="https://api.mapbox.com/mapbox-gl-js/v2.15.0/mapbox-gl.js"></script>',
1055+
),
1056+
).toBe(true);
1057+
});
1058+
1059+
it("detects hyperframes adapter hooks", () => {
1060+
expect(detectWebGlUsage("<script>window.__hfTypegpuTime = 0;</script>")).toBe(true);
1061+
expect(detectWebGlUsage("<script>window.__hfMapbox = map;</script>")).toBe(true);
1062+
expect(detectWebGlUsage("<script>window.__hfMaplibre = map;</script>")).toBe(true);
1063+
});
1064+
1065+
it("does not false-positive on 2d canvas or standard HTML", () => {
1066+
expect(detectWebGlUsage('<script>const ctx = canvas.getContext("2d");</script>')).toBe(false);
1067+
expect(detectWebGlUsage('<div class="hero"><h1>Title</h1><p>Description</p></div>')).toBe(
1068+
false,
1069+
);
1070+
expect(detectWebGlUsage('<script>gsap.to(".box", { x: 100, duration: 1 });</script>')).toBe(
1071+
false,
1072+
);
1073+
});
1074+
});
1075+
10081076
describe("detectAncestorBackgroundImage", () => {
10091077
const wrap = (headCss: string, bodyAttrs = "", inner = "") =>
10101078
`<!doctype html><html><head><style>${headCss}</style></head><body${bodyAttrs ? ` ${bodyAttrs}` : ""}>` +

‎packages/producer/src/services/htmlCompiler.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,8 @@ export interface CompiledComposition {
127127
usesMixBlendMode: boolean;
128128
/** Ancestors of the composition root carry a background-image (gradient/url). */
129129
hasAncestorBackgroundImage: boolean;
130+
/** Composition exercises WebGL / WebGPU accelerated canvas contexts. */
131+
usesWebGl: boolean;
130132
}
131133

132134
const INFERRED_MEDIA_DURATION_ATTR = "data-hf-inferred-duration";
@@ -390,6 +392,22 @@ function detectMixBlendModeUsage(html: string): boolean {
390392
return MIX_BLEND_MODE_PATTERN.test(html);
391393
}
392394

395+
/**
396+
* Signals that the composition exercises WebGL or WebGPU contexts.
397+
* drawElementImage cannot faithfully capture accelerated canvases — they freeze
398+
* under seek-based capture or drop texture presentations, triggering the
399+
* engine's `webgl-context` init-time risk gate. Detecting WebGL statically
400+
* allows the orchestrator to route to the platform baseline (screenshot)
401+
* across parallel workers up front instead of falling into the single-worker
402+
* inversion.
403+
*/
404+
const WEBGL_USAGE_PATTERN =
405+
/\.getContext\s*\(\s*["'](?:webgl2?|experimental-webgl|webgpu)["']|\b(?:THREE\.)?WebGL(?:1|2)?Renderer\b|\bWebGL2?RenderingContext\b|\bdata-requires-webgpu\b|\b__hfTypegpuTime\b|\b__hfMapbox\b|\b__hfMaplibre\b|\bwindow\.THREE\b|\bTHREE\.(?:Scene|PerspectiveCamera|OrthographicCamera|DefaultLoadingManager)\b|\b(?:new\s+)?BABYLON\.Engine\b|\b(?:new\s+)?PIXI\.(?:Application|Renderer)\b|<script[^>]*src=["'][^"']*(?:three|pixi|babylon|mapbox-gl|maplibre-gl)(?:[.-]min)?\.js/i;
406+
407+
export function detectWebGlUsage(html: string): boolean {
408+
return WEBGL_USAGE_PATTERN.test(html);
409+
}
410+
393411
/** A background declaration whose value paints an image (gradient or url). */
394412
const BACKGROUND_IMAGE_DECL_PATTERN =
395413
/(?:^|;|\{)\s*background(?:-image)?\s*:[^;}]*(?:\bgradient\s*\(|url\s*\()/i;
@@ -2035,6 +2053,7 @@ export async function compileForRender(
20352053
const usesThreeDTransforms = detectThreeDTransformUsage(sanitizedHtml);
20362054
const usesMixBlendMode = detectMixBlendModeUsage(sanitizedHtml);
20372055
const hasAncestorBackgroundImage = detectAncestorBackgroundImage(sanitizedHtml);
2056+
let usesWebGl = detectWebGlUsage(sanitizedHtml);
20382057

20392058
const normalizedFontHtml = normalizeSystemFontPrimaryFamilies(
20402059
injectTextRenderingRule(
@@ -2051,6 +2070,9 @@ export async function compileForRender(
20512070

20522071
// CDN scripts are inlined after coalescing so they stay separate from the merged inline runs.
20532072
const assembledHtml = await inlineExternalScripts(coalescedHtml);
2073+
if (!usesWebGl && detectWebGlUsage(assembledHtml)) {
2074+
usesWebGl = true;
2075+
}
20542076

20552077
// Inject studio position seek re-apply script when positions are baked into HTML.
20562078
// GSAP overwrites the `translate` CSS property on every frame seek; this script
@@ -2214,6 +2236,7 @@ export async function compileForRender(
22142236
usesThreeDTransforms,
22152237
usesMixBlendMode,
22162238
hasAncestorBackgroundImage,
2239+
usesWebGl,
22172240
};
22182241
}
22192242

‎packages/producer/src/services/render/stages/compileStage.test.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,67 @@ describe("runCompileStage — master bus", () => {
276276
});
277277
});
278278

279+
const WEBGL_HTML = `<!doctype html>
280+
<html>
281+
<head><meta charset="utf-8"></head>
282+
<body>
283+
<div data-composition-id="root" data-width="1920" data-height="1080" data-duration="1">
284+
<canvas id="c"></canvas>
285+
<script>
286+
const gl = document.getElementById("c").getContext("webgl");
287+
</script>
288+
</div>
289+
</body>
290+
</html>`;
291+
292+
describe("runCompileStage — drawElement WebGL gate", () => {
293+
it("gates drawElement when composition uses WebGL", async () => {
294+
const fixture = setupFixture(WEBGL_HTML);
295+
try {
296+
const cfg = createCfg({ useDrawElement: true });
297+
const result = await runCompileStage({
298+
projectDir: join(fixture.workDir, "project"),
299+
workDir: fixture.workDir,
300+
htmlPath: fixture.htmlPath,
301+
entryFile: "index.html",
302+
job: createJob(),
303+
cfg,
304+
needsAlpha: false,
305+
log: noopLog,
306+
assertNotAborted: () => {},
307+
});
308+
expect(result.compiled.usesWebGl).toBe(true);
309+
expect(result.deCompileGate).toBe("webgl");
310+
expect(cfg.useDrawElement).toBe(false);
311+
} finally {
312+
fixture.cleanup();
313+
}
314+
});
315+
316+
it("does not gate drawElement on standard non-WebGL HTML", async () => {
317+
const fixture = setupFixture(PLAIN_HTML);
318+
try {
319+
const cfg = createCfg({ useDrawElement: true });
320+
const result = await runCompileStage({
321+
projectDir: join(fixture.workDir, "project"),
322+
workDir: fixture.workDir,
323+
htmlPath: fixture.htmlPath,
324+
entryFile: "index.html",
325+
job: createJob(),
326+
cfg,
327+
needsAlpha: false,
328+
log: noopLog,
329+
assertNotAborted: () => {},
330+
});
331+
expect(result.compiled.usesWebGl).toBe(false);
332+
expect(result.deCompileGate).toBeUndefined();
333+
expect(cfg.useDrawElement).toBe(true);
334+
} finally {
335+
fixture.cleanup();
336+
}
337+
});
338+
});
339+
279340
describe("runCompileStage — font fetch cancellation", () => {
280341
let fixture: CompileFixture | null = null;
281342

‎packages/producer/src/services/render/stages/compileStage.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -280,6 +280,26 @@ export async function runCompileStage(input: CompileStageInput): Promise<Compile
280280
"so page-side compositing stays available.",
281281
);
282282
}
283+
// WebGL / accelerated canvas gate: drawElementImage cannot faithfully capture
284+
// accelerated canvases (they freeze under seek-based capture or drop texture
285+
// swaps, triggering the engine's `webgl-context` init-time risk gate).
286+
// When a composition uses WebGL, disable drawElementImage so the render uses
287+
// the platform's baseline route (screenshot) across parallel workers instead
288+
// of collapsing to a single worker under the DE priority inversion.
289+
// HF_FAST_CAPTURE_WEBGL=true or HF_FAST_CAPTURE_CSSFX=true bypasses for R&D.
290+
if (
291+
cfg.useDrawElement &&
292+
process.env.HF_FAST_CAPTURE_WEBGL !== "true" &&
293+
process.env.HF_FAST_CAPTURE_CSSFX !== "true" &&
294+
compiled.usesWebGl
295+
) {
296+
cfg.useDrawElement = false;
297+
deCompileGate = "webgl";
298+
log.info(
299+
"[Render] Fast capture: composition uses WebGL / accelerated canvas, so disabling drawElementImage " +
300+
"for this render. Capture uses the platform's baseline route.",
301+
);
302+
}
283303
// Whenever a compile-time gate cleared useDrawElement, restore page-side
284304
// compositing — but ONLY when resolveConfig auto-disabled it because
285305
// drawElement was going to run. An explicit enablePageSideCompositing:false

‎packages/producer/src/services/renderOrchestrator.test.ts‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ import {
4848
scanElementTags,
4949
envInt,
5050
isDeParallelRouterEnabled,
51+
isWebGlRouteActive,
5152
mergeWorkerInitObservability,
5253
resolveCompositionElementCount,
5354
detectAdaptersStatic,
@@ -2744,6 +2745,9 @@ describe("shouldPreferSingleWorkerDrawElement (DE priority inversion)", () => {
27442745
it("requires drawElement to be enabled and ungated", () => {
27452746
expect(shouldPreferSingleWorkerDrawElement({ ...eligible, useDrawElement: false })).toBe(false);
27462747
expect(shouldPreferSingleWorkerDrawElement({ ...eligible, deCompileGate: "3d" })).toBe(false);
2748+
expect(shouldPreferSingleWorkerDrawElement({ ...eligible, deCompileGate: "webgl" })).toBe(
2749+
false,
2750+
);
27472751
expect(shouldPreferSingleWorkerDrawElement({ ...eligible, forceScreenshot: true })).toBe(false);
27482752
});
27492753

@@ -2757,6 +2761,64 @@ describe("shouldPreferSingleWorkerDrawElement (DE priority inversion)", () => {
27572761
it("is a no-op when workers already resolved to 1", () => {
27582762
expect(shouldPreferSingleWorkerDrawElement({ ...eligible, workerCount: 1 })).toBe(false);
27592763
});
2764+
2765+
it("does not invert a WebGL composition of >= 900 frames to one worker (#5604)", () => {
2766+
// Compositions of >= 900 frames with WebGL must not invert to a single worker.
2767+
// They are routed via layeredOrEffectRoute (from isWebGlRouteActive) or deCompileGate,
2768+
// preserving multi-worker screenshot parallelism.
2769+
const webGlComp = {
2770+
...eligible,
2771+
totalFrames: 900,
2772+
layeredOrEffectRoute: isWebGlRouteActive({ usesWebGl: true }),
2773+
};
2774+
expect(webGlComp.layeredOrEffectRoute).toBe(true);
2775+
expect(shouldPreferSingleWorkerDrawElement(webGlComp)).toBe(false);
2776+
2777+
// Also blocks when caught via runtime WebGL adapter hooks:
2778+
const adapterComp = {
2779+
...eligible,
2780+
totalFrames: 900,
2781+
layeredOrEffectRoute: isWebGlRouteActive({ hasWebGlAdapter: true }),
2782+
};
2783+
expect(adapterComp.layeredOrEffectRoute).toBe(true);
2784+
expect(shouldPreferSingleWorkerDrawElement(adapterComp)).toBe(false);
2785+
2786+
// Bypass via HF_FAST_CAPTURE_WEBGL=true restores the old routing:
2787+
const bypassed = isWebGlRouteActive({
2788+
usesWebGl: true,
2789+
env: { HF_FAST_CAPTURE_WEBGL: "true" },
2790+
});
2791+
expect(bypassed).toBe(false);
2792+
expect(
2793+
shouldPreferSingleWorkerDrawElement({
2794+
...eligible,
2795+
totalFrames: 900,
2796+
layeredOrEffectRoute: bypassed,
2797+
}),
2798+
).toBe(true);
2799+
});
2800+
});
2801+
2802+
describe("isWebGlRouteActive", () => {
2803+
it("returns true when usesWebGl or hasWebGlAdapter is set and bypasses are not active", () => {
2804+
expect(isWebGlRouteActive({ usesWebGl: true, env: {} })).toBe(true);
2805+
expect(isWebGlRouteActive({ hasWebGlAdapter: true, env: {} })).toBe(true);
2806+
expect(isWebGlRouteActive({ usesWebGl: true, hasWebGlAdapter: true, env: {} })).toBe(true);
2807+
});
2808+
2809+
it("returns false when neither usesWebGl nor hasWebGlAdapter is set", () => {
2810+
expect(isWebGlRouteActive({ usesWebGl: false, hasWebGlAdapter: false, env: {} })).toBe(false);
2811+
expect(isWebGlRouteActive({ env: {} })).toBe(false);
2812+
});
2813+
2814+
it("returns false when bypassed by HF_FAST_CAPTURE_WEBGL=true or HF_FAST_CAPTURE_CSSFX=true", () => {
2815+
expect(isWebGlRouteActive({ usesWebGl: true, env: { HF_FAST_CAPTURE_WEBGL: "true" } })).toBe(
2816+
false,
2817+
);
2818+
expect(
2819+
isWebGlRouteActive({ hasWebGlAdapter: true, env: { HF_FAST_CAPTURE_CSSFX: "true" } }),
2820+
).toBe(false);
2821+
});
27602822
});
27612823

27622824
describe("resolveInversionRetryPlan (self-verify retry rollback)", () => {
@@ -2952,6 +3014,7 @@ describe("shouldPreferParallelDrawElement (DE parallel router)", () => {
29523014
it("requires drawElement to be enabled and ungated", () => {
29533015
expect(shouldPreferParallelDrawElement({ ...eligible, useDrawElement: false })).toBe(false);
29543016
expect(shouldPreferParallelDrawElement({ ...eligible, deCompileGate: "3d" })).toBe(false);
3017+
expect(shouldPreferParallelDrawElement({ ...eligible, deCompileGate: "webgl" })).toBe(false);
29553018
expect(shouldPreferParallelDrawElement({ ...eligible, forceScreenshot: true })).toBe(false);
29563019
});
29573020

0 commit comments

Comments
 (0)