Skip to content

Commit 5658f70

Browse files
fix(core): color grading applies to canvas compositions (#5860)
* fix(core): color grading applies to canvas compositions data-color-grading on a <canvas> was ignored by the runtime and refused by lint, the Studio panel and the CLI writer. One source-tag list in the grading contract now feeds all of them. A canvas is sized from its drawing buffer and re-uploaded on every seek and transport tick. * fix(core): keep the comment share of grading files at the base
1 parent 70fbe32 commit 5658f70

13 files changed

Lines changed: 149 additions & 50 deletions

File tree

‎docs/guides/color-grading.mdx‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -166,8 +166,9 @@ Grading picks pixels by value, never by position: no face tracking, region
166166
tracking, rotoscoping or masks — an HSL selection is a color qualifier, not a
167167
shape. To treat one part of a frame, split it into its own media layer and grade
168168
that layer, as in the [implementation reference](/reference/color-grading). It
169-
targets media elements only, so a whole scene including HTML text cannot be
170-
graded.
169+
targets `<video>`, `<img>` and `<canvas>` elements only, so a whole scene
170+
including HTML text cannot be graded. A `<canvas>` is graded at its drawing-buffer
171+
size (`width`/`height`) and re-read on every seek and playback frame.
171172

172173
SDR images and video are the supported case; 4K works at a higher preview and
173174
render cost. An HDR source — iPhone HDR, HLG, Dolby Vision-style — gets an SDR

‎docs/packages/cli.mdx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -481,7 +481,7 @@ npx hyperframes media-treatment --selector '#hero' --clear
481481
| `--all` | Print the exhaustive capability catalog |
482482
| `--project <dir>` | Project directory (default: current directory) |
483483
| `--file <path>` | Composition file (default `index.html`) |
484-
| `--selector <css>` | CSS selector; every matched `<img>` or `<video>` gets the patch |
484+
| `--selector <css>` | CSS selector; every matched `<img>`, `<video>` or `<canvas>` gets the patch |
485485
| `--selector-index <n>` | Zero-based match index to patch only one of several matches |
486486
| `--grading <json>` | Validated colour-grading patch |
487487
| `--apply` | Apply the patch |

‎docs/reference/lint-rules.mdx‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -554,9 +554,11 @@ Source: `packages/lint/src/rules/media.ts`.
554554

555555
### `color_grading_non_media`
556556

557-
**Error.** `data-color-grading` is set on an element other than `<video>` or `<img>`, such as a wrapper or an element with a CSS background. The shader runtime only grades real `<video>` and `<img>` elements, so the attribute has no effect.
557+
**Error.** `data-color-grading` is set on an element other than `<video>`, `<img>` or `<canvas>`, such as a
558+
wrapper or an element with a CSS background. The shader runtime only grades real `<video>`, `<img>` and
559+
`<canvas>` elements, so the attribute has no effect.
558560

559-
**Fix:** Move the attribute to the `<video>` or `<img>` element itself.
561+
**Fix:** Move the attribute to the `<video>`, `<img>` or `<canvas>` element itself.
560562

561563
### `duplicate_media_discovery_risk`
562564

‎packages/cli/src/commands/media-treatment.test.ts‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -598,7 +598,20 @@ describe("applyMediaTreatmentToHtml", () => {
598598
selector: "#hero",
599599
grading: { preset: "warm-daylight" },
600600
}),
601-
).toThrow(/requires an <img> or <video>/);
601+
).toThrow(/requires a <video>, <img> or <canvas>/);
602+
});
603+
604+
it("grades a canvas", () => {
605+
const result = applyMediaTreatmentToHtml(
606+
`<canvas id="drawn" width="640" height="360"></canvas>`,
607+
{
608+
selector: "#drawn",
609+
grading: { preset: "warm-daylight" },
610+
},
611+
);
612+
613+
expect(result.tag).toBe("canvas");
614+
expect(result.html).toContain("data-color-grading=");
602615
});
603616

604617
it("rejects unknown keys instead of silently dropping agent mistakes", () => {

‎packages/cli/src/commands/media-treatment.ts‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import {
1010
serializeHfColorGrading,
1111
} from "@hyperframes/core";
1212
import {
13+
COLOR_GRADING_SOURCE_TAGS,
1314
isColorGradingVariableRef,
1415
validateColorGradingContract,
1516
} from "@hyperframes/parsers/color-grading-contract";
@@ -344,9 +345,11 @@ interface ApplyMediaTreatmentOptions {
344345
clear?: boolean;
345346
}
346347

348+
type GradingSourceTag = (typeof COLOR_GRADING_SOURCE_TAGS)[number];
349+
347350
interface MediaTreatmentTargetResult {
348351
selectorIndex: number;
349-
tag: "img" | "video";
352+
tag: GradingSourceTag;
350353
changed: boolean;
351354
value: string | null;
352355
before: unknown;
@@ -443,7 +446,7 @@ function queryIncludingTemplates(root: Document | Element, selector: string): El
443446
return [];
444447
}
445448

446-
type MediaTarget = { element: Element; selectorIndex: number; tag: "img" | "video" };
449+
type MediaTarget = { element: Element; selectorIndex: number; tag: GradingSourceTag };
447450

448451
function selectMediaElements(
449452
document: Document,
@@ -463,9 +466,11 @@ function selectMediaElements(
463466
const indexes = selectorIndex === undefined ? matches.keys() : [selectorIndex];
464467
return Array.from(indexes, (index) => {
465468
const element = matches[index]!;
466-
const tag = element.tagName.toLowerCase();
467-
if (tag !== "img" && tag !== "video") {
468-
throw new Error(`Color grading requires an <img> or <video>; selector matched <${tag}>`);
469+
const tag = element.tagName.toLowerCase() as GradingSourceTag;
470+
if (!COLOR_GRADING_SOURCE_TAGS.includes(tag)) {
471+
throw new Error(
472+
`Color grading requires a <video>, <img> or <canvas>; selector matched <${tag}>`,
473+
);
469474
}
470475
if (!isHTMLElement(element)) {
471476
throw new Error(`Color grading requires an HTML <${tag}>; selector matched one inside <svg>`);

‎packages/core/src/colorGrading.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
COLOR_GRADING_LUT_KEYS,
1414
COLOR_GRADING_MAX_CURVE_POINTS,
1515
COLOR_GRADING_MAX_SECONDARIES,
16+
COLOR_GRADING_SOURCE_TAGS,
1617
COLOR_GRADING_TOP_LEVEL_KEYS,
1718
COLOR_GRADING_WHEEL_KEYS,
1819
} from "@hyperframes/parsers/color-grading-contract";
@@ -41,6 +42,7 @@ export const COLOR_GRADING_AUTHORED_OPACITY_ATTR = "data-hf-authored-opacity";
4142
export const HF_COLOR_GRADING_CANVAS_ID_PREFIX = "__hf_color_grading_";
4243

4344
export const HF_COLOR_GRADING_COLOR_SPACE = COLOR_GRADING_COLOR_SPACE;
45+
export const HF_COLOR_GRADING_SOURCE_TAGS = COLOR_GRADING_SOURCE_TAGS;
4446

4547
export type HfColorGradingPresetId =
4648
| "neutral"

‎packages/core/src/editing/affordances.test.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,11 @@ describe("resolveEditingAffordances — sections", () => {
172172
expect(s).toMatchObject({ media: true, colorGrading: true });
173173
});
174174

175+
it("canvas: colorGrading", () => {
176+
const s = resolveEditingAffordances(baseFacts({ tag: "canvas" })).sections;
177+
expect(s).toMatchObject({ media: false, colorGrading: true });
178+
});
179+
175180
it("editable text on a plain element: text section", () => {
176181
const s = resolveEditingAffordances(baseFacts({ hasEditableText: true })).sections;
177182
expect(s.text).toBe(true);

‎packages/core/src/editing/affordances.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { HF_AUDIO_GROUP_TAG } from "../audioGroups.js";
2+
import { HF_COLOR_GRADING_SOURCE_TAGS } from "../colorGrading.js";
23
/**
34
* Pure, DOM-free editing-affordance resolution. Single source of truth for what
45
* the studio's edit panel (and any SDK consumer) surfaces per selected element:
@@ -221,7 +222,7 @@ export function resolveEditingSections(facts: EditableElementFacts): EditingSect
221222
media: facts.tag === "video" || facts.tag === "audio" || facts.tag === "img",
222223
audioFx:
223224
facts.tag === "audio" || isAudioBus || (facts.tag === "video" && facts.hasAudio === true),
224-
colorGrading: facts.tag === "video" || facts.tag === "img",
225+
colorGrading: (HF_COLOR_GRADING_SOURCE_TAGS as readonly string[]).includes(facts.tag),
225226
// A bus has no clip range at all — no `data-start`, no duration, and its
226227
// automation clock is composition time — so Start/Duration/End would be
227228
// editing nothing. Audio keeps its timing: an `<audio>` clip is placed on

‎packages/core/src/runtime/colorGrading.test.ts‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -856,6 +856,54 @@ describe("createColorGradingRuntime", () => {
856856
expect(lastUniform1f).toHaveBeenCalledWith("u_blur", 0.7);
857857
});
858858

859+
describe("a graded canvas", () => {
860+
function makeGradedCanvas(width: number, height: number): HTMLCanvasElement {
861+
const canvas = document.createElement("canvas");
862+
canvas.id = "drawn";
863+
canvas.width = width;
864+
canvas.height = height;
865+
canvas.setAttribute(
866+
HF_COLOR_GRADING_ATTR,
867+
serializeHfColorGrading({ adjust: { exposure: 0.5 } }),
868+
);
869+
Object.defineProperty(canvas, "offsetWidth", { value: 640, configurable: true });
870+
Object.defineProperty(canvas, "offsetHeight", { value: 360, configurable: true });
871+
canvas.getBoundingClientRect = () =>
872+
({ width: 640, height: 360, left: 0, top: 0, right: 640, bottom: 360 }) as DOMRect;
873+
document.body.appendChild(canvas);
874+
return canvas;
875+
}
876+
const canvasUploads = (canvas: HTMLCanvasElement) =>
877+
texImage2DCalls.filter((args) => args.length === 6 && args[5] === canvas).length;
878+
879+
it("grades its pixels and re-uploads them on every transport tick and seek", () => {
880+
const canvas = makeGradedCanvas(320, 180);
881+
runtime = createColorGradingRuntime();
882+
883+
expect(runtime.isGraded(canvas)).toBe(true);
884+
expect(runtime.getStatus("drawn")).toMatchObject({ state: "active" });
885+
expect(canvas.style.getPropertyValue("opacity")).toBe("0");
886+
expect(canvasUploads(canvas)).toBe(1);
887+
888+
expect(runtime.redrawAnimated()).toBe(1);
889+
expect(runtime.redraw()).toBe(1);
890+
expect(canvasUploads(canvas)).toBe(3);
891+
});
892+
893+
it("waits for a drawing buffer, whatever its CSS box", () => {
894+
const canvas = makeGradedCanvas(0, 0);
895+
runtime = createColorGradingRuntime();
896+
897+
expect(runtime.getStatus("drawn")).toMatchObject({ state: "pending" });
898+
expect(canvasUploads(canvas)).toBe(0);
899+
900+
canvas.width = 320;
901+
canvas.height = 180;
902+
expect(runtime.redraw()).toBe(1);
903+
expect(canvasUploads(canvas)).toBe(1);
904+
});
905+
});
906+
859907
it("redraws animated held video frames without duplicating active video draws", () => {
860908
const video = makeDrawableVideo();
861909
video.style.setProperty("--hf-color-grading-blur", "0.1");

‎packages/core/src/runtime/colorGrading.ts‎

Lines changed: 40 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import {
55
HF_COLOR_GRADING_CANVAS_ID_PREFIX,
66
HF_COLOR_GRADING_DETAIL_KEYS,
77
HF_COLOR_GRADING_EFFECT_KEYS,
8+
HF_COLOR_GRADING_SOURCE_TAGS,
89
hasHfColorGradingHueCurveValues,
910
hasHfColorGradingRgbCurveValues,
1011
hasHfColorGradingSecondaryValues,
@@ -42,9 +43,15 @@ import { copyMediaVisualStyles } from "../inline-scripts/parityContract";
4243
import { readVariablesForElement } from "./variableScope";
4344
import { swallow } from "./diagnostics";
4445
import { findInjectedRenderFrame } from "./renderFrameSibling";
45-
import { isElementNode, isHtmlElement, isImageElement, isVideoElement } from "./domRealm";
46+
import {
47+
isCanvasElement,
48+
isElementNode,
49+
isHtmlElement,
50+
isImageElement,
51+
isVideoElement,
52+
} from "./domRealm";
4653

47-
type ColorGradingMediaElement = HTMLVideoElement | HTMLImageElement;
54+
type ColorGradingMediaElement = HTMLVideoElement | HTMLImageElement | HTMLCanvasElement;
4855

4956
type EntrySource = "attribute" | "live";
5057

@@ -1462,8 +1469,15 @@ const KUWAHARA_RESOLVE_FRAGMENT_SHADER = [
14621469
"}",
14631470
].join("\n");
14641471

1472+
const COLOR_GRADING_SOURCE_SELECTOR = HF_COLOR_GRADING_SOURCE_TAGS.map(
1473+
(tag) => `${tag}[${HF_COLOR_GRADING_ATTR}]`,
1474+
).join(", ");
1475+
14651476
function isColorGradingMediaElement(value: Element): value is ColorGradingMediaElement {
1466-
return isVideoElement(value) || isImageElement(value);
1477+
return (
1478+
isHtmlElement(value) &&
1479+
(HF_COLOR_GRADING_SOURCE_TAGS as readonly string[]).includes(value.localName)
1480+
);
14671481
}
14681482

14691483
function isVisibleForColorGrading(element: ColorGradingMediaElement): boolean {
@@ -1781,7 +1795,7 @@ function hashStringSeed(value: string): number {
17811795
function seedForElement(element: ColorGradingMediaElement): number {
17821796
const key =
17831797
element.id ||
1784-
element.currentSrc ||
1798+
(!isCanvasElement(element) && element.currentSrc) ||
17851799
element.getAttribute("src") ||
17861800
`${element.tagName}:${Array.prototype.indexOf.call(element.parentNode?.children ?? [], element)}`;
17871801
return hashStringSeed(key);
@@ -2462,31 +2476,22 @@ function resolveTarget(
24622476
}
24632477

24642478
function readSourceSize(source: TexImageSource): { width: number; height: number } | null {
2465-
if (isVideoElement(source)) {
2466-
return source.videoWidth > 0 && source.videoHeight > 0
2467-
? { width: source.videoWidth, height: source.videoHeight }
2468-
: null;
2469-
}
2470-
if (isImageElement(source)) {
2471-
return source.naturalWidth > 0 && source.naturalHeight > 0
2472-
? { width: source.naturalWidth, height: source.naturalHeight }
2473-
: null;
2474-
}
2475-
return null;
2479+
const [width, height] = isVideoElement(source)
2480+
? [source.videoWidth, source.videoHeight]
2481+
: isImageElement(source)
2482+
? [source.naturalWidth, source.naturalHeight]
2483+
: isCanvasElement(source)
2484+
? [source.width, source.height]
2485+
: [0, 0];
2486+
return width > 0 && height > 0 ? { width, height } : null;
24762487
}
24772488

24782489
function isDrawableSource(source: TexImageSource): boolean {
2479-
if (isVideoElement(source)) {
2480-
return (
2481-
source.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA &&
2482-
source.videoWidth > 0 &&
2483-
source.videoHeight > 0
2484-
);
2485-
}
2486-
if (isImageElement(source)) {
2487-
return source.complete && source.naturalWidth > 0 && source.naturalHeight > 0;
2490+
if (isVideoElement(source) && source.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
2491+
return false;
24882492
}
2489-
return false;
2493+
if (isImageElement(source) && !source.complete) return false;
2494+
return readSourceSize(source) !== null;
24902495
}
24912496

24922497
function findRenderFrameImage(video: HTMLVideoElement): HTMLImageElement | null {
@@ -3468,6 +3473,14 @@ function addListener(
34683473
entry.cleanup.push(() => target.removeEventListener(type, listener));
34693474
}
34703475

3476+
function redrawsOnTransportTick(element: ColorGradingMediaElement): boolean {
3477+
if (isCanvasElement(element)) return true;
3478+
if (isVideoElement(element) && !element.paused && !element.ended) return false;
3479+
return ANIMATED_GRADING_PROPERTIES.some(
3480+
(property) => readAnimatedValue(element, property) !== null,
3481+
);
3482+
}
3483+
34713484
function cancelScheduledFrame(entry: ColorGradingEntry): void {
34723485
if (entry.animationFrame !== null) {
34733486
window.cancelAnimationFrame(entry.animationFrame);
@@ -3759,9 +3772,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {
37593772
const refresh = (): number => {
37603773
if (destroyed) return 0;
37613774
const attributeElements = new Set<ColorGradingMediaElement>();
3762-
const nodes = document.querySelectorAll(
3763-
`video[${HF_COLOR_GRADING_ATTR}], img[${HF_COLOR_GRADING_ATTR}]`,
3764-
);
3775+
const nodes = document.querySelectorAll(COLOR_GRADING_SOURCE_SELECTOR);
37653776
nodes.forEach((node) => {
37663777
if (!isColorGradingMediaElement(node)) return;
37673778
attributeElements.add(node);
@@ -3821,12 +3832,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {
38213832
for (const element of trackedElements) {
38223833
const entry = entries.get(element);
38233834
if (!entry) continue;
3824-
const hasAnimatedProperty = ANIMATED_GRADING_PROPERTIES.some(
3825-
(property) => readAnimatedValue(element, property) !== null,
3826-
);
3827-
if (!hasAnimatedProperty) continue;
3828-
if (isVideoElement(element) && !element.paused && !element.ended) continue;
3829-
if (drawEntry(entry)) drawn += 1;
3835+
if (redrawsOnTransportTick(element) && drawEntry(entry)) drawn += 1;
38303836
}
38313837
return drawn;
38323838
};

0 commit comments

Comments
 (0)