Skip to content

Commit eb35629

Browse files
miguel-heygenmiguel-heygen
authored
feat(preview): an unplayable video shows its own frame while its copy is made (#6589)
* feat(preview): an unplayable video shows its own frame while its copy is made * fix(preview): drop the comment on the copy's light flag * fix(preview): shorten the still helper's doc comment * test(preview): each proxy-wait test starts with no copy report left over * fix(preview): cached still frames are swept with the proxy copies
1 parent b22af8f commit eb35629

15 files changed

Lines changed: 341 additions & 26 deletions

‎packages/core/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,7 @@ export {
291291
PREVIEW_PROXY_PRIORITY_HEADER,
292292
PREVIEW_PROXY_PROGRESS_HEADER,
293293
PREVIEW_PROXY_RETRY_PARAM,
294+
PREVIEW_PROXY_STILL_PARAM,
294295
PREVIEW_QUALITY_LEVELS,
295296
formatPreviewProxyBox,
296297
isHeavyForPreview,

‎packages/core/src/previewProxyBox.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ export const PREVIEW_PROXY_BOX_PARAM = "hf-proxy-box";
44
export const PREVIEW_PROXY_LIGHT_PARAM = "hf-proxy-light";
55

66
export const PREVIEW_PROXY_RETRY_PARAM = "hf-proxy-retry";
7+
8+
export const PREVIEW_PROXY_STILL_PARAM = "hf-proxy-still";
79
export const PREVIEW_PROXY_PROGRESS_HEADER = "X-HF-Proxy-Progress";
810
export const PREVIEW_PROXY_PRIORITY_HEADER = "X-HF-Proxy-Priority";
911

‎packages/core/src/runtime/media.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,11 @@ export function isActiveAtPlayhead(el: HTMLMediaElement): boolean {
258258
return lastRelativeTime.has(el);
259259
}
260260

261+
/** The source time the clip shows at the playhead, while it is inside its window. */
262+
export function sourceTimeAtPlayhead(el: HTMLMediaElement): number | undefined {
263+
return lastRelativeTime.get(el);
264+
}
265+
261266
/** Test-only seam: whether any per-source sync state is still tracked for `el`. */
262267
export function hasMediaSyncStateForTest(el: HTMLMediaElement): boolean {
263268
return (

‎packages/core/src/runtime/mediaProxy.test.ts‎

Lines changed: 44 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ vi.mock("./bridge", () => ({
2525

2626
import { postRuntimeMessage } from "./bridge";
2727
import { resetSeekDispatchState, waitForSeekCompletion } from "./adapters/seek-dispatch";
28+
import { resetCopyReporting } from "./previewCopies";
2829

2930
const postRuntimeMessageMock = vi.mocked(postRuntimeMessage);
3031

@@ -100,6 +101,7 @@ beforeEach(() => {
100101

101102
afterEach(() => {
102103
resetSeekDispatchState();
104+
resetCopyReporting();
103105
vi.useRealTimers();
104106
vi.unstubAllGlobals();
105107
vi.clearAllMocks();
@@ -651,6 +653,39 @@ describe("waiting for the proxy copy", () => {
651653
);
652654
});
653655

656+
it("stands in the video's own frame as its poster while the copy is made, then drops it", async () => {
657+
vi.useFakeTimers();
658+
stubProxyRoute(202, 202, 206);
659+
const active = createVideo("/active.mp4");
660+
const idle = createVideo("/idle.mp4");
661+
const clip = (el: HTMLVideoElement) => ({
662+
el,
663+
start: 0,
664+
mediaStart: 4,
665+
duration: 10,
666+
end: 10,
667+
volume: null,
668+
playbackRate: 1,
669+
loop: false,
670+
sourceDuration: null,
671+
});
672+
syncRuntimeMedia({ clips: [clip(active)], timeSeconds: 1, playing: false, playbackRate: 1 });
673+
swapToProxy(active, HEVC_ENTRY, "proactive");
674+
swapToProxy(idle, HEVC_ENTRY, "proactive");
675+
await vi.advanceTimersByTimeAsync(0);
676+
677+
const poster = new URL(active.getAttribute("poster")!, document.baseURI);
678+
expect(poster.pathname).toBe("/active.mp4");
679+
expect(poster.searchParams.get("hf-proxy")).toBe("h264");
680+
expect(poster.searchParams.get("hf-proxy-still")).toBe("5.0");
681+
expect(idle.hasAttribute("poster")).toBe(false);
682+
683+
await vi.advanceTimersByTimeAsync(10_000);
684+
active.dispatchEvent(new Event("loadeddata"));
685+
await vi.advanceTimersByTimeAsync(0);
686+
expect(active.hasAttribute("poster")).toBe(false);
687+
});
688+
654689
it("ignores the original's own metadata and error while the copy is being made", async () => {
655690
vi.useFakeTimers();
656691
window.__HF_MEDIA_CODEC_MAP__ = {};
@@ -663,7 +698,9 @@ describe("waiting for the proxy copy", () => {
663698
handleMetadataForProxy(el);
664699
handleErrorForProxy(el);
665700
await vi.advanceTimersByTimeAsync(0);
666-
expect(postRuntimeMessageMock).not.toHaveBeenCalled();
701+
expect(postRuntimeMessageMock).not.toHaveBeenCalledWith(
702+
expect.objectContaining({ type: "diagnostic" }),
703+
);
667704
expect(route).toHaveBeenCalledTimes(1);
668705

669706
await vi.advanceTimersByTimeAsync(3_000);
@@ -683,7 +720,9 @@ describe("waiting for the proxy copy", () => {
683720

684721
expect(isProxied(el)).toBe(false);
685722
expect(route).toHaveBeenCalledTimes(4);
686-
expect(postRuntimeMessageMock).toHaveBeenCalledTimes(1);
723+
expect(
724+
postRuntimeMessageMock.mock.calls.filter(([message]) => message.type === "diagnostic"),
725+
).toHaveLength(1);
687726
expect(postRuntimeMessageMock).toHaveBeenCalledWith(
688727
expect.objectContaining({
689728
code: "runtime_media_proxy_unavailable",
@@ -761,7 +800,9 @@ describe("waiting for the proxy copy", () => {
761800

762801
expect(route).toHaveBeenCalledTimes(1);
763802
expect(el.load).not.toHaveBeenCalled();
764-
expect(postRuntimeMessageMock).not.toHaveBeenCalled();
803+
expect(postRuntimeMessageMock).not.toHaveBeenCalledWith(
804+
expect.objectContaining({ type: "diagnostic" }),
805+
);
765806
});
766807

767808
it("keeps one wait for a new file even after the dropped wait for the old one wakes", async () => {

‎packages/core/src/runtime/mediaProxy.ts‎

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
import { postRuntimeMessage } from "./bridge";
22
import { swallow } from "./diagnostics";
3-
import { evictMediaSyncState, HOLD_CAP_MS, isActiveAtPlayhead } from "./media";
3+
import {
4+
evictMediaSyncState,
5+
HOLD_CAP_MS,
6+
isActiveAtPlayhead,
7+
sourceTimeAtPlayhead,
8+
} from "./media";
49
import { findInjectedRenderFrame } from "./renderFrameSibling";
510
import type { RuntimeJson } from "./types";
611
import { isVideoElement } from "./domRealm";
@@ -17,6 +22,7 @@ import {
1722
PREVIEW_PROXY_BOX_PARAM,
1823
PREVIEW_PROXY_LIGHT_PARAM,
1924
PREVIEW_PROXY_RETRY_PARAM,
25+
PREVIEW_PROXY_STILL_PARAM,
2026
quantizePreviewProxyBox,
2127
scalePreviewProxyBox,
2228
type PreviewProxyBox,
@@ -193,14 +199,14 @@ function requestBox(el: HTMLMediaElement, copy: BoxedCopy, box: PreviewProxyBox
193199
pendingUpgrades.set(el, href);
194200
const wanted = () =>
195201
pendingUpgrades.get(el) === href && el.isConnected && el.getAttribute("src") === copy.href;
196-
noteCopy(el, "making");
202+
noteCopy(el, "making", null, copy.light);
197203
void waitForServedProxy(
198204
href,
199205
wanted,
200-
(fraction) => wanted() && noteCopy(el, "making", fraction),
206+
(fraction) => wanted() && noteCopy(el, "making", fraction, copy.light),
201207
() => isActiveAtPlayhead(el),
202208
).then((served) => {
203-
if (wanted()) noteCopy(el, served ? "ready" : "failed");
209+
if (wanted()) noteCopy(el, served ? "ready" : "failed", null, copy.light);
204210
if (served && wanted()) loadProxy(el, href, copy.originalSrc, copy.variant, box, copy.light);
205211
if (pendingUpgrades.get(el) === href) pendingUpgrades.delete(el);
206212
});
@@ -390,6 +396,27 @@ function lookupCodecMapEntry<T>(pathname: string, map: Record<string, T>): T | n
390396
);
391397
}
392398

399+
const stills = new WeakSet<HTMLMediaElement>();
400+
401+
/** While a file the browser cannot play is being copied, its own frame at the playhead stands in as the poster. */
402+
function showStill(
403+
el: HTMLMediaElement,
404+
originalSrc: string,
405+
variant: string,
406+
box: PreviewProxyBox | null,
407+
): void {
408+
const at = sourceTimeAtPlayhead(el);
409+
if (at === undefined || stills.has(el) || el.hasAttribute("poster")) return;
410+
const url = new URL(appendProxyParam(originalSrc, variant, box), document.baseURI);
411+
url.searchParams.set(PREVIEW_PROXY_STILL_PARAM, at.toFixed(1));
412+
stills.add(el);
413+
el.setAttribute("poster", url.href);
414+
}
415+
416+
function dropStill(el: HTMLMediaElement): void {
417+
if (stills.delete(el)) el.removeAttribute("poster");
418+
}
419+
393420
function appendProxyParam(
394421
src: string,
395422
variant: string,
@@ -471,26 +498,30 @@ async function makeCopy(el: HTMLMediaElement, job: CopyJob, scale: number | null
471498
lightFacts !== undefined,
472499
retryNext.delete(el),
473500
);
474-
noteCopy(el, "making");
501+
const light = lightFacts !== undefined;
502+
noteCopy(el, "making", null, light);
475503
const served = await waitForServedProxy(
476504
proxiedSrc,
477505
live,
478-
(fraction) => noteCopy(el, "making", fraction),
506+
(fraction) => noteCopy(el, "making", fraction, light),
479507
() => isActiveAtPlayhead(el),
508+
false,
509+
light ? undefined : () => showStill(el, originalSrc, variant, box),
480510
);
481511
if (!live()) {
512+
dropStill(el);
482513
if (proxyRequested.get(el) === originalAttr) proxyRequested.delete(el);
483514
if (lightFacts && el.isConnected && el.getAttribute("src") === originalAttr) {
484515
applyPreviewQuality(el);
485516
}
486517
return;
487518
}
488519
if (!served) {
489-
noteCopy(el, "failed");
520+
noteCopy(el, "failed", null, light);
490521
emitUnavailableDiagnostic(el, "proxy_playback_failed", originalSrc);
491522
return;
492523
}
493-
noteCopy(el, "ready");
524+
noteCopy(el, "ready", null, light);
494525
swappedElements.set(el, originalAttr);
495526
if (lightFacts) lightElements.add(el);
496527
loadProxy(el, proxiedSrc, originalSrc, variant, box, lightFacts !== undefined);
@@ -499,7 +530,7 @@ async function makeCopy(el: HTMLMediaElement, job: CopyJob, scale: number | null
499530
return new Promise((landed) => {
500531
el.addEventListener("loadeddata", landed, { once: true });
501532
el.addEventListener("error", landed, { once: true });
502-
});
533+
}).then(() => dropStill(el));
503534
}
504535

505536
/**

‎packages/core/src/runtime/previewCopies.test.ts‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import { afterEach, describe, expect, it, vi } from "vitest";
2+
3+
vi.mock("./bridge", () => ({ postRuntimeMessage: vi.fn() }));
4+
5+
import { postRuntimeMessage } from "./bridge";
6+
import { copiesReady, noteCopy, setCopyReportingLevel } from "./previewCopies";
7+
8+
const posted = vi.mocked(postRuntimeMessage);
9+
const video = () => document.body.appendChild(document.createElement("video"));
10+
11+
afterEach(() => {
12+
setCopyReportingLevel(1);
13+
document.body.innerHTML = "";
14+
posted.mockClear();
15+
});
16+
17+
describe("copy reporting", () => {
18+
it("counts the copy a file cannot play without at Full quality, and ignores a lighter one", () => {
19+
setCopyReportingLevel(1);
20+
posted.mockClear();
21+
noteCopy(video(), "making", 0.5, true);
22+
expect(posted).not.toHaveBeenCalled();
23+
24+
noteCopy(video(), "making", 0.5, false);
25+
expect(posted).toHaveBeenCalledWith(
26+
expect.objectContaining({
27+
type: "preview-copies",
28+
copies: expect.objectContaining({ total: 1, ready: 0, progress: 0.5 }),
29+
}),
30+
);
31+
});
32+
33+
it("keeps the unplayable file's copy when the level changes, and leaves lighter readiness alone", () => {
34+
setCopyReportingLevel(1);
35+
noteCopy(video(), "making", null, false);
36+
expect(copiesReady()).toBe(true);
37+
38+
setCopyReportingLevel(0.5);
39+
noteCopy(video(), "making", null, true);
40+
expect(copiesReady()).toBe(false);
41+
setCopyReportingLevel(1);
42+
expect(copiesReady()).toBe(true);
43+
expect(posted).toHaveBeenLastCalledWith(
44+
expect.objectContaining({ copies: expect.objectContaining({ total: 1 }) }),
45+
);
46+
});
47+
});

‎packages/core/src/runtime/previewCopies.ts‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
import { postRuntimeMessage } from "./bridge";
22

3-
type CopyState = { state: "making" | "ready" | "failed"; progress: number | null };
3+
type CopyState = {
4+
state: "making" | "ready" | "failed";
5+
progress: number | null;
6+
light: boolean;
7+
};
48

59
const REPORT_EVERY_MS = 1000;
610
const copies = new Map<HTMLMediaElement, CopyState>();
@@ -12,7 +16,7 @@ let dirtyWhileWaiting = false;
1216
function report(): boolean {
1317
for (const el of copies.keys()) if (!el.isConnected) copies.delete(el);
1418
const states = [...copies.values()];
15-
const wanted = reporting && states.length > 0;
19+
const wanted = states.length > 0;
1620
if (!wanted && !reportedAny) return false;
1721
reportedAny = wanted;
1822
const made = states.reduce(
@@ -50,23 +54,33 @@ function scheduleReport(): void {
5054
}, REPORT_EVERY_MS);
5155
}
5256

53-
/** Starts counting the copies wanted at a level; at full quality none are wanted. */
57+
/** Starts counting the lighter copies wanted at a level; at full quality none are wanted. */
5458
export function setCopyReportingLevel(level: number): void {
55-
copies.clear();
59+
for (const [el, copy] of copies) if (copy.light) copies.delete(el);
5660
reporting = level < 1;
5761
clearTimeout(timer);
5862
timer = undefined;
5963
dirtyWhileWaiting = false;
6064
scheduleReport();
6165
}
6266

67+
export function resetCopyReporting(): void {
68+
copies.clear();
69+
clearTimeout(timer);
70+
timer = undefined;
71+
reporting = false;
72+
reportedAny = false;
73+
dirtyWhileWaiting = false;
74+
}
75+
6376
export function noteCopy(
6477
el: HTMLMediaElement,
6578
state: CopyState["state"],
6679
progress: number | null = null,
80+
light = true,
6781
): void {
68-
if (!reporting) return;
69-
copies.set(el, { state, progress });
82+
if (light && !reporting) return;
83+
copies.set(el, { state, progress, light });
7084
scheduleReport();
7185
}
7286

@@ -76,5 +90,5 @@ export function failedCopyElements(): HTMLMediaElement[] {
7690

7791
/** True when every lighter copy wanted at this level is ready; true when none are wanted. */
7892
export function copiesReady(): boolean {
79-
return [...copies.values()].every((copy) => copy.state === "ready");
93+
return [...copies.values()].every((copy) => !copy.light || copy.state === "ready");
8094
}

‎packages/core/src/runtime/proxyWait.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,12 @@ function withoutRetryFlag(href: string): string {
1212
return url.href;
1313
}
1414

15-
function askHeaders(atPlayhead: boolean, background: boolean): Record<string, string> {
15+
function askHeaders(
16+
atPlayhead: boolean,
17+
background: boolean,
18+
onAsk?: () => void,
19+
): Record<string, string> {
20+
onAsk?.();
1621
const headers: Record<string, string> = { Range: "bytes=0-0" };
1722
if (atPlayhead) headers[PREVIEW_PROXY_PRIORITY_HEADER] = "1";
1823
else if (background) headers[PREVIEW_PROXY_PRIORITY_HEADER] = "0";
@@ -34,11 +39,12 @@ export async function waitForServedProxy(
3439
onProgress?: (fraction: number) => void,
3540
atPlayhead: () => boolean = () => false,
3641
background = false,
42+
onAsk?: () => void,
3743
): Promise<boolean> {
3844
let failed = 0;
3945
let ask = href;
4046
while (live()) {
41-
const headers = askHeaders(atPlayhead(), background);
47+
const headers = askHeaders(atPlayhead(), background, onAsk);
4248
const res = await fetch(ask, { headers, cache: "no-store" }).catch(() => null);
4349
ask = withoutRetryFlag(href);
4450
void res?.body?.cancel().catch(() => {});

‎packages/studio-server/src/helpers/mediaCodecMap.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,8 @@ export const PROXY_VARIANT_CONFIG: Record<
159159
vp8: { extension: ".webm", contentType: "video/webm" },
160160
};
161161

162+
export const PROXY_STILL_EXTENSION = ".jpg";
163+
162164
export function isProxyVariant(value: string): value is ProxyVariant {
163165
return Object.hasOwn(PROXY_VARIANT_CONFIG, value);
164166
}

0 commit comments

Comments
 (0)