Skip to content

Commit 84fd701

Browse files
feat(studio): preview copy progress, retry and a remembered quality (#6366)
* feat(studio-server): a copy being made says how far it has got, and can be retried * feat(core): the preview reports its lighter copies' progress and retries failed ones * feat(studio): the preview quality and its lighter copies' progress live in the player store * feat(studio): every host forwards the preview quality, and the choice is kept per computer * feat(studio): the preview quality knows if a person or the app chose it * fix(core): the first report of the lighter copies is not held back by the throttle * test(studio): the preview quality preference test writes under the real key * fix(core): a quality change reports its copies at once, not after the throttle * style: format the preview quality files * feat(core): report how playback is going, and what the preview holds, to the host * fix(studio): the preview quality levels come from a browser-safe core entry * style: format the new entry's files * style: drop two comments the file's ratchet and the citation check refuse * feat(core): tell which lower levels would give a video a lighter copy * fix(studio): drop the old level's playback health when the level changes * feat(studio-server): preview facts carry the probed codec of every video * refactor(studio): route the preview's reports through a table for the complexity gate * fix(core): a copy's retry flag is sent on the first ask only, not on every poll * fix(core): type the health report and restore the player hook's complexity directive * style: cut four comments the comment ratchet refuses * fix(studio): keep playerStore.ts within its 600-line cap * fix(core): a scene mounting does not restart the playback health window * fix(core): the playback health window restarts only when the display scale changes * fix(core): a repeated display scale no longer restarts the playback health window * fix(studio): route the preview's reports by an explicit list of message types
1 parent 3f1950b commit 84fd701

31 files changed

Lines changed: 1264 additions & 19 deletions

‎packages/core/package-subpaths.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,12 @@
122122
"types": "./dist/motionBlurSettings.d.ts",
123123
"environments": ["browser", "bun", "node"]
124124
},
125+
"./preview-quality": {
126+
"source": "./src/previewProxyBox.ts",
127+
"runtime": "./dist/previewProxyBox.js",
128+
"types": "./dist/previewProxyBox.d.ts",
129+
"environments": ["browser", "bun", "node"]
130+
},
125131
"./gsap-cdn": {
126132
"source": "./src/gsapCdn.ts",
127133
"runtime": "./dist/gsapCdn.js",

‎packages/core/package.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,12 @@
134134
"import": "./src/motionBlurSettings.ts",
135135
"types": "./src/motionBlurSettings.ts"
136136
},
137+
"./preview-quality": {
138+
"bun": "./src/previewProxyBox.ts",
139+
"node": "./dist/previewProxyBox.js",
140+
"import": "./src/previewProxyBox.ts",
141+
"types": "./src/previewProxyBox.ts"
142+
},
137143
"./gsap-cdn": {
138144
"bun": "./src/gsapCdn.ts",
139145
"node": "./dist/gsapCdn.js",
@@ -650,6 +656,10 @@
650656
"import": "./dist/motionBlurSettings.js",
651657
"types": "./dist/motionBlurSettings.d.ts"
652658
},
659+
"./preview-quality": {
660+
"import": "./dist/previewProxyBox.js",
661+
"types": "./dist/previewProxyBox.d.ts"
662+
},
653663
"./gsap-cdn": {
654664
"import": "./dist/gsapCdn.js",
655665
"types": "./dist/gsapCdn.d.ts"

‎packages/core/src/index.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,8 @@ export {
273273
HEAVY_PIXEL_SHARE,
274274
PREVIEW_PROXY_BOX_PARAM,
275275
PREVIEW_PROXY_LIGHT_PARAM,
276+
PREVIEW_PROXY_PROGRESS_HEADER,
277+
PREVIEW_PROXY_RETRY_PARAM,
276278
PREVIEW_QUALITY_LEVELS,
277279
formatPreviewProxyBox,
278280
isHeavyForPreview,

‎packages/core/src/lightEntries.test.ts‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,10 @@ function loadedModules(entry: string): string[] {
2222
}
2323

2424
describe("entries a host imports for one helper", () => {
25-
it.each(["./safe-path", "./asset-paths"])("%s loads none of the parsers", (entry) => {
26-
expect(loadedModules(entry).filter((path) => HEAVY.test(path))).toEqual([]);
27-
});
25+
it.each(["./safe-path", "./asset-paths", "./preview-quality"])(
26+
"%s loads none of the parsers",
27+
(entry) => {
28+
expect(loadedModules(entry).filter((path) => HEAVY.test(path))).toEqual([]);
29+
},
30+
);
2831
});

‎packages/core/src/previewProxyBox.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ export const PREVIEW_PROXY_BOX_PARAM = "hf-proxy-box";
33

44
export const PREVIEW_PROXY_LIGHT_PARAM = "hf-proxy-light";
55

6+
export const PREVIEW_PROXY_RETRY_PARAM = "hf-proxy-retry";
7+
export const PREVIEW_PROXY_PROGRESS_HEADER = "X-HF-Proxy-Progress";
8+
69
export type PreviewProxyBox = { width: number; height: number };
710

811
// 256 * 2^(k/4), even: nearby boxes share one cached copy, and a rung overshoots by at most ~19%.
@@ -48,6 +51,7 @@ export type PreviewSourceFacts = {
4851
height: number;
4952
bitDepth: number;
5053
bitrate: number;
54+
codec?: string;
5155
};
5256

5357
export const HEAVY_PIXEL_SHARE = 0.5;

‎packages/core/src/runtime/bridge.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ type BridgeDeps = {
3434
onClearRuntimeData?: (channel: string, requestId?: number) => void;
3535
onSetDisplayScale?: (scale: number) => void;
3636
onSetPreviewQuality?: (level: number) => void;
37+
onRetryPreviewCopies?: () => void;
38+
onRequestPreviewFacts?: () => void;
3739
getCanonicalFps: () => number;
3840
};
3941

@@ -78,6 +80,8 @@ const CONTROL_HANDLERS = new Map<string, ControlHandler>(
7880
"set-idle-heartbeat": (data, deps) => deps.onSetIdleHeartbeat(Boolean(data.slow)),
7981
"set-display-scale": (data, deps) => deps.onSetDisplayScale?.(Number(data.scale)),
8082
"set-preview-quality": (data, deps) => deps.onSetPreviewQuality?.(Number(data.level)),
83+
"retry-preview-copies": (_d, deps) => deps.onRetryPreviewCopies?.(),
84+
"request-preview-facts": (_d, deps) => deps.onRequestPreviewFacts?.(),
8185
"set-root-duration": (data, deps) => deps.onSetRootDuration(Number(data.durationSeconds ?? 0)),
8286
"set-play-range": (data, deps) =>
8387
deps.onSetPlayRange(

‎packages/core/src/runtime/init.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ import {
2424
lengthIsAuthored,
2525
stopMediaDownload,
2626
} from "./preloadMedia";
27+
import { notePlaybackFrame, noteDisplayScale } from "./playbackHealth";
28+
import { copiesReady } from "./previewCopies";
2729
import { installRuntimeControlBridge, postRuntimeMessage, setRuntimeProtocolFps } from "./bridge";
2830
import { instantTolerance } from "../clipFacts";
2931
import { isInClipWindow } from "./clipWindow";
@@ -95,6 +97,9 @@ import {
9597
handleMetadataForProxy,
9698
isRenderMode,
9799
maybeProxyProactively,
100+
describeVideoSources,
101+
levelsWithLighterCopies,
102+
retryPreviewCopies,
98103
setPreviewQuality,
99104
setProxyDisplayScale,
100105
} from "./mediaProxy";
@@ -5619,6 +5624,20 @@ export function initSandboxRuntimeModular(): void {
56195624

56205625
const t = clock.now();
56215626
state.currentTime = t;
5627+
if (!window.__HF_EXPORT_RENDER_SEEK_CONFIG) {
5628+
notePlaybackFrame({
5629+
nowMs: performance.now(),
5630+
filmTime: t,
5631+
fps: state.canonicalFps,
5632+
rate: state.playbackRate,
5633+
playing,
5634+
hidden: document.visibilityState === "hidden",
5635+
waiting: mediaClipsInWindow.some(
5636+
({ el }) => el.tagName === "VIDEO" && !el.paused && el.readyState < 3,
5637+
),
5638+
layoutKey: `${window.innerWidth}x${window.innerHeight}@${window.devicePixelRatio}`,
5639+
});
5640+
}
56225641
// During a paused Studio manual-edit drag, the draft writer owns the
56235642
// gestured element's transform (e.g. gsap.set for x/y). Re-seeking the
56245643
// timeline every frame re-applies the animated value and clobbers the
@@ -5974,8 +5993,23 @@ export function initSandboxRuntimeModular(): void {
59745993
onSetDisplayScale: (scale) => {
59755994
setProxyDisplayScale(scale);
59765995
setPreviewRasterScale(scale);
5996+
noteDisplayScale(scale);
59775997
},
5998+
onRequestPreviewFacts: () =>
5999+
postRuntimeMessage({
6000+
source: "hf-preview",
6001+
type: "preview-facts",
6002+
facts: {
6003+
sceneCount: windowScenes.length,
6004+
mountedElementCount: document.body.querySelectorAll("*").length,
6005+
durationSeconds: clock.getDuration(),
6006+
videos: describeVideoSources(),
6007+
lighterAt: levelsWithLighterCopies(),
6008+
copiesReady: copiesReady(),
6009+
},
6010+
}),
59786011
onSetPreviewQuality: setPreviewQuality,
6012+
onRetryPreviewCopies: retryPreviewCopies,
59796013
onSetIdleHeartbeat: (slow) => {
59806014
slowIdleHeartbeat = slow;
59816015
wakeTransport();

‎packages/core/src/runtime/mediaProxy.test.ts‎

Lines changed: 136 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ const syncRuntimeMedia = (
1212
import {
1313
deriveCodecMapKey,
1414
handleErrorForProxy,
15+
describeVideoSources,
1516
handleMetadataForProxy,
1617
maybeProxyProactively,
1718
swapToProxy,
@@ -791,6 +792,21 @@ describe("a copy sized to how large the video shows", () => {
791792
const boxOf = (el: HTMLMediaElement) =>
792793
new URL(el.src, document.baseURI).searchParams.get("hf-proxy-box");
793794

795+
it("names the lower levels at which a video would get a lighter copy", async () => {
796+
const { setProxyDisplayScale } = await framedVideo(1080, 1920);
797+
const { levelsWithLighterCopies } = await import("./mediaProxy");
798+
const facts = { width: 1080, height: 1920, bitDepth: 8, bitrate: 0 };
799+
window.__HF_MEDIA_SOURCE_FACTS__ = { "/video.mp4": facts };
800+
expect(levelsWithLighterCopies()).toEqual([]);
801+
802+
setProxyDisplayScale(1);
803+
expect(levelsWithLighterCopies()).toEqual([0.25, 0.125]);
804+
805+
window.__HF_MEDIA_SOURCE_FACTS__ = { "/video.mp4": { ...facts, bitDepth: 10 } };
806+
expect(levelsWithLighterCopies()).toEqual([0.5, 0.25, 0.125]);
807+
delete window.__HF_MEDIA_SOURCE_FACTS__;
808+
});
809+
794810
it("asks only once the host says how large it shows the frame, for that size", async () => {
795811
const { el, setProxyDisplayScale } = await framedVideo(1080, 1920);
796812
await settle();
@@ -890,14 +906,15 @@ describe("preview quality: lighter copies of heavy files", () => {
890906
value: 1080,
891907
});
892908
vi.spyOn(window, "devicePixelRatio", "get").mockReturnValue(1);
909+
const bridge = await import("./bridge");
893910
const video = (src = "/video.mp4") => {
894911
const el = createVideo(src);
895912
stubCanPlayType(el, "probably");
896913
el.getBoundingClientRect = () => ({ width: 1920, height: 1080 }) as DOMRect;
897914
return el;
898915
};
899916
mod.setProxyDisplayScale(1);
900-
return { ...mod, video };
917+
return { ...mod, video, posted: vi.mocked(bridge.postRuntimeMessage) };
901918
}
902919

903920
const params = (el: HTMLMediaElement) => new URL(el.src, document.baseURI).searchParams;
@@ -989,6 +1006,95 @@ describe("preview quality: lighter copies of heavy files", () => {
9891006
expect(params(el).get("hf-proxy-box")).toBe("2048x1218");
9901007
});
9911008

1009+
describe("reporting the copies", () => {
1010+
const lastCopies = (posted: typeof postRuntimeMessageMock) =>
1011+
posted.mock.calls
1012+
.map(([message]) => message)
1013+
.filter((message) => message.type === "preview-copies")
1014+
.at(-1);
1015+
1016+
it("counts the copies wanted, with how far the making has got, and says none at full quality", async () => {
1017+
vi.useFakeTimers();
1018+
let reply = new Response(null, {
1019+
status: 202,
1020+
headers: { "Retry-After": "1", "X-HF-Proxy-Progress": "0.5" },
1021+
});
1022+
vi.stubGlobal(
1023+
"fetch",
1024+
vi.fn(async () => reply.clone()),
1025+
);
1026+
const { video, setPreviewQuality, posted } = await stage({ "/video.mp4": HEAVY });
1027+
video();
1028+
1029+
setPreviewQuality(0.5);
1030+
await vi.advanceTimersByTimeAsync(0);
1031+
expect(lastCopies(posted)).toMatchObject({
1032+
copies: { total: 1, ready: 0, failed: 0, progress: 0, error: null },
1033+
});
1034+
1035+
await vi.advanceTimersByTimeAsync(1000);
1036+
expect(lastCopies(posted)).toMatchObject({ copies: { total: 1, ready: 0, progress: 0.5 } });
1037+
1038+
reply = new Response("x", { status: 206 });
1039+
await vi.advanceTimersByTimeAsync(2000);
1040+
expect(lastCopies(posted)).toMatchObject({
1041+
copies: { total: 1, ready: 1, failed: 0, progress: 1, error: null },
1042+
});
1043+
1044+
setPreviewQuality(1);
1045+
expect(lastCopies(posted)).toMatchObject({ copies: null });
1046+
});
1047+
1048+
it("reports a copy that could not be made, and asks again with the retry flag", async () => {
1049+
vi.useFakeTimers();
1050+
vi.stubGlobal(
1051+
"fetch",
1052+
vi.fn(async () => new Response("no", { status: 502 })),
1053+
);
1054+
const { video, setPreviewQuality, retryPreviewCopies, posted } = await stage({
1055+
"/video.mp4": HEAVY,
1056+
});
1057+
const el = video();
1058+
1059+
setPreviewQuality(0.5);
1060+
await vi.advanceTimersByTimeAsync(10_000);
1061+
expect(lastCopies(posted)).toMatchObject({
1062+
copies: { total: 1, failed: 1, error: "failed" },
1063+
});
1064+
expect(isProxied(el)).toBe(false);
1065+
1066+
vi.stubGlobal(
1067+
"fetch",
1068+
vi.fn(async () => new Response("x", { status: 206 })),
1069+
);
1070+
retryPreviewCopies();
1071+
await vi.advanceTimersByTimeAsync(2000);
1072+
1073+
expect(vi.mocked(fetch).mock.calls[0]?.[0]).toContain("hf-proxy-retry=1");
1074+
expect(lastCopies(posted)).toMatchObject({ copies: { total: 1, ready: 1, failed: 0 } });
1075+
expect(isProxied(el)).toBe(true);
1076+
});
1077+
});
1078+
1079+
it("sends the retry flag on the first ask only, so a copy that fails again is not remade on every poll", async () => {
1080+
vi.useFakeTimers();
1081+
const fetchMock = vi.fn(async () => new Response("no", { status: 502 }));
1082+
vi.stubGlobal("fetch", fetchMock);
1083+
const { video, setPreviewQuality, retryPreviewCopies } = await stage({ "/video.mp4": HEAVY });
1084+
video();
1085+
setPreviewQuality(0.5);
1086+
await vi.advanceTimersByTimeAsync(10_000);
1087+
1088+
fetchMock.mockClear();
1089+
retryPreviewCopies();
1090+
await vi.advanceTimersByTimeAsync(10_000);
1091+
1092+
const asked = fetchMock.mock.calls.map(([href]) => String(href));
1093+
expect(asked.length).toBeGreaterThan(1);
1094+
expect(asked[0]).toContain("hf-proxy-retry=1");
1095+
expect(asked.slice(1).some((href) => href.includes("hf-proxy-retry"))).toBe(false);
1096+
});
1097+
9921098
it("ignores a level that is not one of the four", async () => {
9931099
const { video, setPreviewQuality } = await stage({ "/video.mp4": HEAVY });
9941100
const el = video();
@@ -999,3 +1105,32 @@ describe("preview quality: lighter copies of heavy files", () => {
9991105
expect(isProxied(el)).toBe(false);
10001106
});
10011107
});
1108+
1109+
describe("describeVideoSources", () => {
1110+
it("groups the videos by what they are, counting repeats and never naming a file", () => {
1111+
window.__HF_MEDIA_CODEC_MAP__ = {
1112+
"/a.mp4": HEVC_ENTRY,
1113+
"/b.mp4": HEVC_ENTRY,
1114+
"/c.mp4": H264_ENTRY,
1115+
};
1116+
const facts = { width: 3840, height: 2160, bitDepth: 10, bitrate: 0 };
1117+
window.__HF_MEDIA_SOURCE_FACTS__ = {
1118+
"/a.mp4": facts,
1119+
"/b.mp4": facts,
1120+
"/c.mp4": { ...facts, width: 1920, height: 1080, bitDepth: 8 },
1121+
};
1122+
window.__HF_MEDIA_SOURCE_FACTS__["/c.mp4"] = {
1123+
...window.__HF_MEDIA_SOURCE_FACTS__["/c.mp4"],
1124+
codec: "h264",
1125+
};
1126+
window.__HF_MEDIA_CODEC_MAP__ = { "/a.mp4": HEVC_ENTRY, "/b.mp4": HEVC_ENTRY };
1127+
for (const src of ["/a.mp4", "/b.mp4", "/c.mp4", "/d.mp4"]) createVideo(src);
1128+
1129+
expect(describeVideoSources()).toEqual([
1130+
{ codec: "hevc", width: 3840, height: 2160, bitDepth: 10, count: 2 },
1131+
{ codec: "h264", width: 1920, height: 1080, bitDepth: 8, count: 1 },
1132+
{ codec: "unknown", width: 0, height: 0, bitDepth: 0, count: 1 },
1133+
]);
1134+
delete window.__HF_MEDIA_SOURCE_FACTS__;
1135+
});
1136+
});

0 commit comments

Comments
 (0)