Skip to content

Commit 036e41f

Browse files
miguel-heygenmiguel-heygen
authored
fix(core): iPhone Safari keeps one connection per file for the take on screen and the next (#7012)
* fix(core): iPhone Safari keeps one connection per file for the take on screen and the next * fix(core): drop the doc comments the comment ratchet counts in studioPreviewMark
1 parent cd59842 commit 036e41f

6 files changed

Lines changed: 155 additions & 6 deletions

File tree

‎packages/core/src/compositionReadiness.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { createRuntimeStartTimeResolver } from "./runtime/startResolver.js";
22
import { isRuntimeElementVisibleAt } from "./runtime/timeline.js";
33
import type { RuntimeTimelineLike } from "./runtime/types.js";
4-
import { STUDIO_PREVIEW_MARK_META } from "./studioPreviewMark.js";
4+
import { defersMediaIn } from "./studioPreviewMark.js";
55

66
/** A composition is "ready" once every declared input settles, not just once
77
* its duration is known. Each input returns null (nothing to wait on) or a
@@ -104,8 +104,8 @@ export function scanPendingCompositionAssets(
104104
timelineRegistry: runtimeWindow?.__timelines,
105105
includeAuthoredTimingAttrs: true,
106106
});
107-
// Studio's preview buffers a clip off the first frame only once it comes due, as with a lazy image.
108-
const defersMedia = doc.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`) !== null;
107+
// Studio's preview and the player buffer a clip off the first frame only once it comes due, as with a lazy image.
108+
const defersMedia = defersMediaIn(doc);
109109
const pendingMedia = Array.from(doc.querySelectorAll("video, audio"))
110110
.filter(isRealmHtmlMediaElement)
111111
.filter((el) =>

‎packages/core/src/runtime/entry.test.ts‎

Lines changed: 105 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,11 @@
22
import { afterEach, describe, expect, it, vi } from "vitest";
33
import { mediaElementTime } from "./mediaClock";
44
import { HF_COLOR_GRADING_ATTR, serializeHfColorGrading } from "../colorGrading";
5-
import { STUDIO_PREVIEW_LAZY_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
5+
import {
6+
PLAYER_FRAME_ATTR,
7+
STUDIO_PREVIEW_LAZY_ATTR,
8+
STUDIO_PREVIEW_MARK_META,
9+
} from "../studioPreviewMark";
610
import { revealClip } from "./clipReveal";
711
import type { RuntimeTimelineLike } from "./types";
812

@@ -91,6 +95,12 @@ const servePreview = () =>
9195
document.head.appendChild(
9296
Object.assign(document.createElement("meta"), { name: STUDIO_PREVIEW_MARK_META }),
9397
);
98+
const asSafari = () =>
99+
vi
100+
.spyOn(navigator, "userAgent", "get")
101+
.mockReturnValue(
102+
"Mozilla/5.0 AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15",
103+
);
94104
const neverDecodes = (clip: HTMLElement) => {
95105
withImage(clip).querySelector("img")!.decode = () => new Promise<void>(() => {});
96106
return clip;
@@ -429,6 +439,64 @@ describe("runtime entry", () => {
429439
expect(now.getAttribute("src")).toBe("now.mp4");
430440
});
431441

442+
// Three takes of one file, each a picture and a sound, as an edit cut from a single source is.
443+
const takesOfOneFile = () => {
444+
const root = mountRoot();
445+
const take = (start: string) => {
446+
const pair = [timed(root, "video", start), timed(root, "audio", start)];
447+
for (const el of pair) el.setAttribute("src", "/source.mp4");
448+
return pair;
449+
};
450+
return [take("0"), take("3"), take("9")];
451+
};
452+
const inPlayerFrame = (userAgent: "safari" | "chrome") => {
453+
const frame = document.createElement("iframe");
454+
frame.setAttribute(PLAYER_FRAME_ATTR, "");
455+
vi.spyOn(window, "frameElement", "get").mockReturnValue(frame);
456+
if (userAgent === "safari") asSafari();
457+
};
458+
459+
it("in the player on Safari a file keeps the take on screen and the next one loading, and frees the rest", async () => {
460+
spyLoad();
461+
inPlayerFrame("safari");
462+
const [first, second, third] = takesOfOneFile();
463+
464+
await evaluateRuntime();
465+
expect([first, second, third].map((take) => armed(...take))).toEqual([
466+
[true, true],
467+
[true, true],
468+
[false, false],
469+
]);
470+
471+
window.__player?.seek(3.5);
472+
expect([first, second, third].map((take) => armed(...take))).toEqual([
473+
[false, false],
474+
[true, true],
475+
[true, true],
476+
]);
477+
});
478+
479+
it("in the player on Chromium every take of a file loads at parse", async () => {
480+
spyLoad();
481+
inPlayerFrame("chrome");
482+
const takes = takesOfOneFile();
483+
484+
await evaluateRuntime();
485+
expect(takes.map((take) => armed(...take))).toEqual([
486+
[true, true],
487+
[true, true],
488+
[true, true],
489+
]);
490+
});
491+
492+
it("in Studio's preview a clip 5 s ahead stays unloaded, as the look-ahead is 2 s", async () => {
493+
servePreview();
494+
const [now, soon] = videos(mountRoot(), "0", "5");
495+
496+
await evaluateRuntime();
497+
expect(armed(now, soon)).toEqual([true, false]);
498+
});
499+
432500
it("loads a clip with no authored length as a render does, so no reload drops its duration", async () => {
433501
spyLoad();
434502
servePreview();
@@ -940,6 +1008,42 @@ describe("runtime entry", () => {
9401008
expect([first.preload, later.preload, untrimmed.preload]).toEqual(["", "none", ""]);
9411009
});
9421010

1011+
it("gives each parsed clip preload none in the player's frame, where no clip fetches ahead of the first take", async () => {
1012+
const frame = document.createElement("iframe");
1013+
frame.setAttribute(PLAYER_FRAME_ATTR, "");
1014+
vi.spyOn(window, "frameElement", "get").mockReturnValue(frame);
1015+
asSafari();
1016+
const root = mountRoot();
1017+
timed(root, "div", "0");
1018+
await evaluateRuntime();
1019+
const [first, later] = videos(root, "0", "5");
1020+
await Promise.resolve();
1021+
expect([first.preload, later.preload]).toEqual(["", "none"]);
1022+
});
1023+
1024+
it("leaves Chromium loading every clip at parse in the player's frame", async () => {
1025+
const frame = document.createElement("iframe");
1026+
frame.setAttribute(PLAYER_FRAME_ATTR, "");
1027+
vi.spyOn(window, "frameElement", "get").mockReturnValue(frame);
1028+
const root = mountRoot();
1029+
timed(root, "div", "0");
1030+
await evaluateRuntime();
1031+
const [first, later] = videos(root, "0", "5");
1032+
await Promise.resolve();
1033+
expect([first.preload, later.preload]).toEqual(["", ""]);
1034+
});
1035+
1036+
it("leaves a frame the player did not mark loading every clip at parse", async () => {
1037+
vi.spyOn(window, "frameElement", "get").mockReturnValue(document.createElement("iframe"));
1038+
asSafari();
1039+
const root = mountRoot();
1040+
timed(root, "div", "0");
1041+
await evaluateRuntime();
1042+
const [first, later] = videos(root, "0", "5");
1043+
await Promise.resolve();
1044+
expect([first.preload, later.preload]).toEqual(["", ""]);
1045+
});
1046+
9431047
it("defers a clip at 0 inside a later scene, by its start in the film", async () => {
9441048
servePreview();
9451049
const root = mountRoot();

‎packages/core/src/runtime/init.ts‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ import {
4242
skipsHiddenImages,
4343
} from "./timedClipHide";
4444
import {
45+
inPlayerFrame,
4546
PREVIEW_ONLY_ATTRS,
4647
RUNTIME_STAMPED_ATTR,
4748
STUDIO_PREVIEW_FAR_SCENE_ATTR as FAR_SCENE_ATTR,
@@ -2996,6 +2997,29 @@ export function initSandboxRuntimeModular(): void {
29962997
let moveSceneWindow = (_time: number): boolean => false;
29972998
const LOOKAHEAD_SECONDS = 2;
29982999
const RELEASE_LOOKAHEAD_SECONDS = 4;
3000+
// The player keeps a connection per file for the clips on screen and the next take only (see inPlayerFrame).
3001+
const playerFrame = inPlayerFrame(document);
3002+
const NEXT_TAKE_SPAN_SECONDS = 0.5;
3003+
let nextTakes: { t: number; bound: number; clips: Set<HTMLMediaElement> } | null = null;
3004+
const fileOf = (el: HTMLMediaElement) =>
3005+
el.getAttribute("src") ?? el.querySelector("source")?.getAttribute("src") ?? "";
3006+
const nextTakeClips = (t: number): Set<HTMLMediaElement> => {
3007+
if (nextTakes?.t === t && nextTakes.bound === metadataBoundMedia.size) return nextTakes.clips;
3008+
const byFile = new Map<string, { el: HTMLMediaElement; start: number }[]>();
3009+
for (const el of metadataBoundMedia) {
3010+
const file = fileOf(el);
3011+
const start = resolveAbsoluteMediaStartSeconds(el);
3012+
if (!el.isConnected || !file || !(start > t)) continue;
3013+
byFile.set(file, [...(byFile.get(file) ?? []), { el, start }]);
3014+
}
3015+
const clips = new Set<HTMLMediaElement>();
3016+
for (const later of byFile.values()) {
3017+
const next = Math.min(...later.map((clip) => clip.start));
3018+
for (const clip of later) if (clip.start <= next + NEXT_TAKE_SPAN_SECONDS) clips.add(clip.el);
3019+
}
3020+
nextTakes = { t, bound: metadataBoundMedia.size, clips };
3021+
return clips;
3022+
};
29993023
// Unskipped while within the window on either side of the playhead, so a step or shuttle back
30003024
// across a cut finds the clip it left still loaded; the edge checks catch a clip shorter than it.
30013025
const dueSoon = (
@@ -3248,6 +3272,8 @@ export function initSandboxRuntimeModular(): void {
32483272
): number[] | null => {
32493273
const own = windowOf(node);
32503274
if (!own || !Number.isFinite(own.start)) return null;
3275+
// Which clip is next on its file flips at every other clip's start.
3276+
if (playerFrame && isMediaElement(node)) return null;
32513277
const edges: number[] = [];
32523278
for (const ahead of [0, LOOKAHEAD_SECONDS, RELEASE_LOOKAHEAD_SECONDS])
32533279
edges.push(own.start - ahead, own.end - ahead, own.end + ahead);
@@ -3392,7 +3418,9 @@ export function initSandboxRuntimeModular(): void {
33923418
preloadNearPlayhead(
33933419
rawNode,
33943420
isVisibleNow,
3395-
upcoming || mediaStaysDue(rawNode, visibleAt, currentTime),
3421+
playerFrame
3422+
? nextTakeClips(currentTime).has(rawNode)
3423+
: upcoming || mediaStaysDue(rawNode, visibleAt, currentTime),
33963424
currentTime,
33973425
dueAtLoopStart(rawNode, visibleAt),
33983426
);

‎packages/core/src/runtime/timedClipHide.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { CLIP_HIDDEN_ATTR, olderShaderBuildOnPage, revealClip } from "./clipReveal";
22
import {
3+
defersMediaIn,
34
RUNTIME_STAMPED_ATTR,
45
STUDIO_PREVIEW_CAPTURE_MODE,
56
STUDIO_PREVIEW_MARK_META,
@@ -60,7 +61,7 @@ function appendStyle(parent: Element, attr: string, css: string): void {
6061
}
6162

6263
export function defersMedia(): boolean {
63-
return document.querySelector(STUDIO_PREVIEW_MARK) !== null;
64+
return defersMediaIn(document);
6465
}
6566

6667
/** True in Studio's preview, where hidden clips' images are neither fetched nor decoded until due. */

‎packages/core/src/studioPreviewMark.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,18 @@
1+
import { isWebKit } from "./runtime/mediaClock.js";
2+
13
export const STUDIO_PREVIEW_MARK_META = "hyperframes-studio-preview";
24
export const STUDIO_PREVIEW_CAPTURE_MODE = "capture";
5+
export const PLAYER_FRAME_ATTR = "data-hf-player-frame";
6+
7+
export function inPlayerFrame(doc: Document): boolean {
8+
return isWebKit() && doc.defaultView?.frameElement?.hasAttribute(PLAYER_FRAME_ATTR) === true;
9+
}
10+
11+
export function defersMediaIn(doc: Document): boolean {
12+
return (
13+
doc.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`) !== null || inPlayerFrame(doc)
14+
);
15+
}
316
export const STUDIO_PREVIEW_UPCOMING_ATTR = "data-hf-upcoming";
417
export const STUDIO_PREVIEW_LAZY_ATTR = "data-hf-preview-lazy";
518
export const STUDIO_PREVIEW_ERRORS = "__hfPreviewErrors";

‎packages/player/src/iframe-dom.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
* Keeps constructor boilerplate out of the web component class body.
55
*/
66

7+
import { PLAYER_FRAME_ATTR } from "@hyperframes/core/studio-preview-mark";
8+
79
// Cached Constructable Stylesheet shared across all player instances.
810
let _sharedSheet: CSSStyleSheet | null = null;
911

@@ -49,6 +51,7 @@ export function createCompositionIframe(): {
4951
iframe.allow = "autoplay; fullscreen";
5052
iframe.referrerPolicy = "no-referrer";
5153
iframe.title = "HyperFrames Composition";
54+
iframe.setAttribute(PLAYER_FRAME_ATTR, "");
5255

5356
container.appendChild(iframe);
5457
return { container, iframe };

0 commit comments

Comments
 (0)