Skip to content

Commit 9eaf9ef

Browse files
fix(core): a graded clip's canvas no longer comes out blank on a real GPU (#6615)
* fix(core): a graded clip's canvas no longer comes out blank on a real GPU * chore(core): cut the comments the grading fix added * fix(core): a clip that outgrows the GPU's buffer goes back to its source video
1 parent 8b1f88b commit 9eaf9ef

2 files changed

Lines changed: 96 additions & 6 deletions

File tree

‎packages/core/src/runtime/colorGrading.test.ts‎

Lines changed: 71 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,11 @@ LUT_3D_SIZE 2
3131
`;
3232

3333
function createMockWebGl(
34-
options: { failMediaUpload?: boolean; halfFloatSupported?: boolean } = {},
34+
options: {
35+
failMediaUpload?: boolean;
36+
halfFloatSupported?: boolean;
37+
drawingBuffer?: { width: number; height: number };
38+
} = {},
3539
): WebGLRenderingContext {
3640
const shader = {};
3741
const program = {};
@@ -124,6 +128,12 @@ function createMockWebGl(
124128
drawArrays: vi.fn(),
125129
deleteTexture: vi.fn(),
126130
deleteBuffer: vi.fn(),
131+
get drawingBufferWidth() {
132+
return options.drawingBuffer?.width;
133+
},
134+
get drawingBufferHeight() {
135+
return options.drawingBuffer?.height;
136+
},
127137
} as unknown as WebGLRenderingContext;
128138
}
129139

@@ -944,6 +954,66 @@ describe("createColorGradingRuntime", () => {
944954
expect(video.style.getPropertyPriority("opacity")).toBe("important");
945955
});
946956

957+
it("caps a clip shown far larger than the screen to what a GPU can draw, keeping its shape", () => {
958+
vi.stubGlobal("devicePixelRatio", 1);
959+
const video = makeDrawableVideo();
960+
Object.defineProperty(video, "offsetWidth", { value: 6582, configurable: true });
961+
Object.defineProperty(video, "offsetHeight", { value: 11520, configurable: true });
962+
963+
const { canvas } = startRuntimeWithVideo(video);
964+
965+
expect(canvas.width * canvas.height).toBeLessThanOrEqual(1 << 24);
966+
expect(canvas.width / canvas.height).toBeCloseTo(6582 / 11520, 2);
967+
});
968+
969+
it("leaves the source showing when the drawing buffer cannot hold the canvas", () => {
970+
getContextSpy.mockImplementation((type: string) =>
971+
type === "webgl"
972+
? createMockWebGl({ drawingBuffer: { width: 100, height: 100 } })
973+
: type === "2d"
974+
? mock2d()
975+
: null,
976+
);
977+
978+
const { video } = startRuntimeWithVideo();
979+
980+
expect(video.style.getPropertyValue("opacity")).not.toBe("0");
981+
});
982+
983+
it("gives a drawn clip back to its source when a later draw outgrows the drawing buffer", () => {
984+
const drawingBuffer = { width: 4096, height: 4096 };
985+
getContextSpy.mockImplementation((type: string) =>
986+
type === "webgl" ? createMockWebGl({ drawingBuffer }) : type === "2d" ? mock2d() : null,
987+
);
988+
const { video, canvas } = startRuntimeWithVideo();
989+
expect(video.style.getPropertyValue("opacity")).toBe("0");
990+
991+
drawingBuffer.width = 100;
992+
drawingBuffer.height = 100;
993+
Object.defineProperty(video, "offsetWidth", { value: 3000, configurable: true });
994+
video.dispatchEvent(new Event("seeked"));
995+
996+
expect(video.style.getPropertyValue("opacity")).not.toBe("0");
997+
expect(canvas.style.display).toBe("none");
998+
});
999+
1000+
it("draws a paused video again once its frame is presented, after the first draw sampled nothing", () => {
1001+
const presented: Array<() => void> = [];
1002+
const video = makeDrawableVideo();
1003+
Object.defineProperty(video, "paused", { value: true, configurable: true });
1004+
Object.defineProperty(video, "requestVideoFrameCallback", {
1005+
value: (callback: () => void) => presented.push(callback),
1006+
configurable: true,
1007+
});
1008+
1009+
startRuntimeWithVideo(video);
1010+
const drawsBefore = texImage2DCalls.length;
1011+
presented.shift()?.();
1012+
1013+
expect(texImage2DCalls.length).toBeGreaterThan(drawsBefore);
1014+
expect(presented).toHaveLength(0);
1015+
});
1016+
9471017
it("keeps the canvas visible when producer render-frame injection hides the source video", () => {
9481018
const video = makeDrawableVideo();
9491019
Object.defineProperty(video, "readyState", {

‎packages/core/src/runtime/colorGrading.ts‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,8 @@ type LutCacheEntry =
320320
const LUT_CACHE = new Map<string, LutCacheEntry>();
321321
const COLOR_GRADING_CANVAS_ATTR = "data-hf-color-grading-canvas";
322322
const COLOR_GRADING_CANVAS_CLASS = "__hf_color_grading_canvas__";
323+
const MAX_GRADING_CANVAS_PIXELS = 1 << 24;
324+
const MAX_GRADING_CANVAS_SIDE = 8192;
323325

324326
/** Captures authored media opacity before animation or grading can mutate it. */
325327
export function installAuthoredOpacityCapture(): void {
@@ -2900,6 +2902,16 @@ function resolvedLayoutSize(primary: number, fallback: number): number {
29002902
return Math.max(0, Math.round(primary > 0 ? primary : fallback));
29012903
}
29022904

2905+
function fitToGpuLimits(width: number, height: number): { width: number; height: number } {
2906+
const scale = Math.min(
2907+
1,
2908+
Math.sqrt(MAX_GRADING_CANVAS_PIXELS / (width * height)),
2909+
MAX_GRADING_CANVAS_SIDE / Math.max(width, height),
2910+
);
2911+
const fitted = { width: Math.floor(width * scale), height: Math.floor(height * scale) };
2912+
return scale < 1 ? fitted : { width: Math.round(width), height: Math.round(height) };
2913+
}
2914+
29032915
function updateCanvasLayout(
29042916
entry: ColorGradingEntry,
29052917
styleSource: HTMLElement,
@@ -2932,8 +2944,7 @@ function updateCanvasLayout(
29322944
return null;
29332945
}
29342946
const pixelRatio = Math.max(1, window.devicePixelRatio || 1);
2935-
const width = Math.round(cssWidth * pixelRatio);
2936-
const height = Math.round(cssHeight * pixelRatio);
2947+
const { width, height } = fitToGpuLimits(cssWidth * pixelRatio, cssHeight * pixelRatio);
29372948
if (canvas.width !== width) canvas.width = width;
29382949
if (canvas.height !== height) canvas.height = height;
29392950
return { width, height };
@@ -3497,6 +3508,11 @@ function renderGradedFrame(
34973508
frameTime,
34983509
);
34993510
drawFullscreenQuad(gl, { quad: program.quad, position: shader.position });
3511+
if (gl.drawingBufferWidth < layout.width || gl.drawingBufferHeight < layout.height) {
3512+
restoreSourceElement(entry);
3513+
entry.canvas.style.display = "none";
3514+
throw new Error("The GPU's drawing buffer is smaller than the grading canvas");
3515+
}
35003516
presentEntry(entry, layout);
35013517
}
35023518

@@ -3772,10 +3788,14 @@ function installEntryListeners(entry: ColorGradingEntry): void {
37723788
const redraw = () => {
37733789
drawEntry(entry);
37743790
};
3791+
const redrawOnPresentedFrame = () => {
3792+
drawEntry(entry);
3793+
scheduleVideoDraw(entry);
3794+
};
37753795
addListener(entry, entry.element, "load", redraw);
37763796
addListener(entry, entry.element, "loadedmetadata", redraw);
3777-
addListener(entry, entry.element, "loadeddata", redraw);
3778-
addListener(entry, entry.element, "seeked", redraw);
3797+
addListener(entry, entry.element, "loadeddata", redrawOnPresentedFrame);
3798+
addListener(entry, entry.element, "seeked", redrawOnPresentedFrame);
37793799
addListener(entry, entry.element, "timeupdate", redraw);
37803800
addListener(entry, window, "resize", redraw);
37813801
if (isVideoElement(entry.element)) {
@@ -4007,7 +4027,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {
40074027
trackedElements.add(element);
40084028
installEntryListeners(entry);
40094029
drawEntry(entry);
4010-
if (isVideoElement(element) && !element.paused) scheduleVideoDraw(entry);
4030+
if (isVideoElement(element)) scheduleVideoDraw(entry);
40114031
return true;
40124032
};
40134033

0 commit comments

Comments
 (0)