Skip to content

Commit 38d9aeb

Browse files
feat(core): lighter preview copies of heavy videos below full quality (#6365)
* feat(core): tell which video files deserve a lighter preview copy, and at which size * feat(studio-server): serve lighter copies of heavy playable files, a keyframe every second * feat(core): the preview plays lighter copies of heavy videos below full quality * style: shorten two comments * style: shorten a comment * style: shorten a comment * fix(core): format the preview quality files and split the copy sizing for the complexity gate * style: format the studio-server preview files * fix(core): type the copy job's level as a preview quality * chore(producer): raise the package size budget for the preview runtime growth
1 parent c0ddd32 commit 38d9aeb

19 files changed

Lines changed: 721 additions & 118 deletions

‎packages/core/src/index.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,11 +268,20 @@ export {
268268
type ToneMapSourceColour,
269269
} from "./hdrToneMap";
270270
export {
271+
HEAVY_BITRATE,
272+
HEAVY_PIXEL_SHARE,
271273
PREVIEW_PROXY_BOX_PARAM,
274+
PREVIEW_PROXY_LIGHT_PARAM,
275+
PREVIEW_QUALITY_LEVELS,
272276
formatPreviewProxyBox,
277+
isHeavyForPreview,
278+
isPreviewQuality,
273279
parsePreviewProxyBox,
274280
quantizePreviewProxyBox,
281+
scalePreviewProxyBox,
275282
type PreviewProxyBox,
283+
type PreviewQuality,
284+
type PreviewSourceFacts,
276285
} from "./previewProxyBox";
277286

278287
// Inline scripts

‎packages/core/src/previewProxyBox.test.ts‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
import { describe, expect, it } from "vitest";
2-
import { parsePreviewProxyBox, quantizePreviewProxyBox } from "./previewProxyBox";
2+
import {
3+
isHeavyForPreview,
4+
isPreviewQuality,
5+
parsePreviewProxyBox,
6+
quantizePreviewProxyBox,
7+
scalePreviewProxyBox,
8+
type PreviewSourceFacts,
9+
} from "./previewProxyBox";
310

411
describe("preview proxy box", () => {
512
it("rounds each side up to the next rung", () => {
@@ -28,3 +35,34 @@ describe("preview proxy box", () => {
2835
expect(parsePreviewProxyBox("")).toBeNull();
2936
});
3037
});
38+
39+
describe("lighter copies of files Chrome can play", () => {
40+
const full: PreviewSourceFacts = { width: 1920, height: 1080, bitDepth: 8, bitrate: 10_000_000 };
41+
const box = { width: 1218, height: 724 };
42+
const shown = { width: 1448, height: 862 };
43+
44+
it("multiplies the shown box by the level, on the ladder", () => {
45+
expect(scalePreviewProxyBox(box, 1)).toEqual(box);
46+
expect(scalePreviewProxyBox(box, 0.5)).toEqual({ width: 724, height: 362 });
47+
expect(scalePreviewProxyBox(box, 0.125)).toEqual({ width: 256, height: 256 });
48+
});
49+
50+
it("calls a file heavy for a much larger source, 10-bit or deeper, or 40 Mbit/s and up", () => {
51+
expect(isHeavyForPreview(full, shown)).toBe(false);
52+
expect(isHeavyForPreview({ ...full, width: 3840, height: 2160 }, shown)).toBe(true);
53+
expect(isHeavyForPreview({ ...full, bitDepth: 10 }, shown)).toBe(true);
54+
expect(isHeavyForPreview({ ...full, bitrate: 39_999_999 }, shown)).toBe(false);
55+
expect(isHeavyForPreview({ ...full, bitrate: 40_000_000 }, shown)).toBe(true);
56+
});
57+
58+
it("calls a source heavy when the copy box has half its pixels or fewer", () => {
59+
const half = { width: 1448, height: 724 };
60+
expect(isHeavyForPreview({ ...full, width: 2896, height: 1448 }, half)).toBe(true);
61+
expect(isHeavyForPreview({ ...full, width: 1448, height: 1000 }, half)).toBe(false);
62+
});
63+
64+
it("knows the four levels and nothing else", () => {
65+
expect([1, 0.5, 0.25, 0.125].every(isPreviewQuality)).toBe(true);
66+
expect([0, 0.75, 2, "1", null].some(isPreviewQuality)).toBe(false);
67+
});
68+
});

‎packages/core/src/previewProxyBox.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
/** Query parameter carrying the largest box, in device pixels, a preview shows a proxied video at. */
22
export const PREVIEW_PROXY_BOX_PARAM = "hf-proxy-box";
33

4+
export const PREVIEW_PROXY_LIGHT_PARAM = "hf-proxy-light";
5+
46
export type PreviewProxyBox = { width: number; height: number };
57

68
// 256 * 2^(k/4), even: nearby boxes share one cached copy, and a rung overshoots by at most ~19%.
@@ -33,3 +35,32 @@ export function parsePreviewProxyBox(raw: string): PreviewProxyBox | null {
3335
const height = Number(match[2]);
3436
return RUNGS.includes(width) && RUNGS.includes(height) ? { width, height } : null;
3537
}
38+
39+
export const PREVIEW_QUALITY_LEVELS = [1, 0.5, 0.25, 0.125] as const;
40+
export type PreviewQuality = (typeof PREVIEW_QUALITY_LEVELS)[number];
41+
42+
export function isPreviewQuality(value: unknown): value is PreviewQuality {
43+
return PREVIEW_QUALITY_LEVELS.some((level) => level === value);
44+
}
45+
46+
export type PreviewSourceFacts = {
47+
width: number;
48+
height: number;
49+
bitDepth: number;
50+
bitrate: number;
51+
};
52+
53+
export const HEAVY_PIXEL_SHARE = 0.5;
54+
export const HEAVY_BITRATE = 40_000_000;
55+
56+
export function scalePreviewProxyBox(box: PreviewProxyBox, level: PreviewQuality): PreviewProxyBox {
57+
return quantizePreviewProxyBox(box.width * level, box.height * level) ?? box;
58+
}
59+
60+
export function isHeavyForPreview(facts: PreviewSourceFacts, copyBox: PreviewProxyBox): boolean {
61+
return (
62+
copyBox.width * copyBox.height <= HEAVY_PIXEL_SHARE * facts.width * facts.height ||
63+
facts.bitDepth > 8 ||
64+
facts.bitrate >= HEAVY_BITRATE
65+
);
66+
}

‎packages/core/src/runtime/bridge.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ type BridgeDeps = {
3333
onSetRuntimeData?: (channel: string, payload: unknown, requestId?: number) => void;
3434
onClearRuntimeData?: (channel: string, requestId?: number) => void;
3535
onSetDisplayScale?: (scale: number) => void;
36+
onSetPreviewQuality?: (level: number) => void;
3637
getCanonicalFps: () => number;
3738
};
3839

@@ -76,6 +77,7 @@ const CONTROL_HANDLERS = new Map<string, ControlHandler>(
7677
"set-playback-rate": (data, deps) => deps.onSetPlaybackRate(Number(data.playbackRate ?? 1)),
7778
"set-idle-heartbeat": (data, deps) => deps.onSetIdleHeartbeat(Boolean(data.slow)),
7879
"set-display-scale": (data, deps) => deps.onSetDisplayScale?.(Number(data.scale)),
80+
"set-preview-quality": (data, deps) => deps.onSetPreviewQuality?.(Number(data.level)),
7981
"set-root-duration": (data, deps) => deps.onSetRootDuration(Number(data.durationSeconds ?? 0)),
8082
"set-play-range": (data, deps) =>
8183
deps.onSetPlayRange(

‎packages/core/src/runtime/init.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ import {
9595
handleMetadataForProxy,
9696
isRenderMode,
9797
maybeProxyProactively,
98+
setPreviewQuality,
9899
setProxyDisplayScale,
99100
} from "./mediaProxy";
100101
import {
@@ -5973,6 +5974,7 @@ export function initSandboxRuntimeModular(): void {
59735974
setProxyDisplayScale(scale);
59745975
setPreviewRasterScale(scale);
59755976
},
5977+
onSetPreviewQuality: setPreviewQuality,
59765978
onSetIdleHeartbeat: (slow) => {
59775979
slowIdleHeartbeat = slow;
59785980
wakeTransport();

‎packages/core/src/runtime/mediaProxy.test.ts‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -869,3 +869,133 @@ describe("a copy sized to how large the video shows", () => {
869869
expect(boxOf(el)).toBe("2436x4096");
870870
});
871871
});
872+
873+
describe("preview quality: lighter copies of heavy files", () => {
874+
const HEAVY = { width: 3840, height: 2160, bitDepth: 8, bitrate: 10_000_000 };
875+
const LIGHT = { width: 1280, height: 720, bitDepth: 8, bitrate: 10_000_000 };
876+
877+
// A framed page showing a 1920x1080 stage at full size on a 1x screen: a 2048x1218 box.
878+
async function stage(facts: Record<string, typeof HEAVY>, codecMap?: Record<string, unknown>) {
879+
vi.resetModules();
880+
vi.spyOn(window, "parent", "get").mockReturnValue({} as Window);
881+
const mod = await import("./mediaProxy");
882+
window.__HF_MEDIA_SOURCE_FACTS__ = facts;
883+
if (codecMap) window.__HF_MEDIA_CODEC_MAP__ = codecMap as never;
884+
Object.defineProperty(document.documentElement, "clientWidth", {
885+
configurable: true,
886+
value: 1920,
887+
});
888+
Object.defineProperty(document.documentElement, "clientHeight", {
889+
configurable: true,
890+
value: 1080,
891+
});
892+
vi.spyOn(window, "devicePixelRatio", "get").mockReturnValue(1);
893+
const video = (src = "/video.mp4") => {
894+
const el = createVideo(src);
895+
stubCanPlayType(el, "probably");
896+
el.getBoundingClientRect = () => ({ width: 1920, height: 1080 }) as DOMRect;
897+
return el;
898+
};
899+
mod.setProxyDisplayScale(1);
900+
return { ...mod, video };
901+
}
902+
903+
const params = (el: HTMLMediaElement) => new URL(el.src, document.baseURI).searchParams;
904+
905+
afterEach(() => {
906+
delete (window as { __HF_MEDIA_SOURCE_FACTS__?: unknown }).__HF_MEDIA_SOURCE_FACTS__;
907+
});
908+
909+
it("changes nothing at full quality", async () => {
910+
const { video, setPreviewQuality, maybeProxyProactively } = await stage({
911+
"/video.mp4": HEAVY,
912+
});
913+
const el = video();
914+
maybeProxyProactively(el);
915+
setPreviewQuality(1);
916+
await settle();
917+
918+
expect(isProxied(el)).toBe(false);
919+
expect(fetch).not.toHaveBeenCalled();
920+
});
921+
922+
it("swaps a heavy file Chrome can play to a copy at the shown box times the level", async () => {
923+
const { video, setPreviewQuality } = await stage({ "/video.mp4": HEAVY });
924+
const el = video();
925+
926+
setPreviewQuality(0.5);
927+
await settle();
928+
929+
expect(params(el).get("hf-proxy")).toBe("auto");
930+
expect(params(el).get("hf-proxy-box")).toBe("1024x724");
931+
expect(params(el).get("hf-proxy-light")).toBe("1");
932+
});
933+
934+
it("leaves a file alone that is not heavy at that box", async () => {
935+
const { video, setPreviewQuality } = await stage({ "/video.mp4": LIGHT });
936+
const el = video();
937+
938+
setPreviewQuality(0.5);
939+
await settle();
940+
941+
expect(isProxied(el)).toBe(false);
942+
});
943+
944+
it("puts the original back at full quality", async () => {
945+
const { video, setPreviewQuality } = await stage({ "/video.mp4": HEAVY });
946+
const el = video();
947+
setPreviewQuality(0.5);
948+
await settle();
949+
expect(isProxied(el)).toBe(true);
950+
951+
setPreviewQuality(1);
952+
953+
expect(el.getAttribute("src")).toBe("/video.mp4");
954+
expect(el.load).toHaveBeenCalledTimes(2);
955+
});
956+
957+
it("lightens a video that appears after the level was lowered", async () => {
958+
const { video, setPreviewQuality, maybeProxyProactively } = await stage({
959+
"/video.mp4": HEAVY,
960+
});
961+
setPreviewQuality(0.25);
962+
const el = video();
963+
964+
maybeProxyProactively(el);
965+
await settle();
966+
967+
expect(params(el).get("hf-proxy-box")).toBe("512x362");
968+
});
969+
970+
it("sizes the copy a file cannot play without by the level too, and resizes it as the level moves", async () => {
971+
const { video, setPreviewQuality, maybeProxyProactively } = await stage(
972+
{ "/video.mp4": HEAVY },
973+
{ "/video.mp4": HEVC_ENTRY },
974+
);
975+
const el = video();
976+
stubCanPlayType(el, "");
977+
maybeProxyProactively(el);
978+
await settle();
979+
expect(params(el).get("hf-proxy-box")).toBe("2048x1218");
980+
expect(params(el).has("hf-proxy-light")).toBe(false);
981+
982+
setPreviewQuality(0.5);
983+
await settle();
984+
expect(params(el).get("hf-proxy-box")).toBe("1024x724");
985+
expect(params(el).has("hf-proxy-light")).toBe(false);
986+
987+
setPreviewQuality(1);
988+
await settle();
989+
expect(params(el).get("hf-proxy-box")).toBe("2048x1218");
990+
});
991+
992+
it("ignores a level that is not one of the four", async () => {
993+
const { video, setPreviewQuality } = await stage({ "/video.mp4": HEAVY });
994+
const el = video();
995+
996+
setPreviewQuality(0.7);
997+
await settle();
998+
999+
expect(isProxied(el)).toBe(false);
1000+
});
1001+
});

0 commit comments

Comments
 (0)