Skip to content

Commit 5973273

Browse files
fix(runtime): resolve media clip start once for playback, manifest, and audio (#3833)
* fix(core): play and draw media clips at the same time A clip placed inside a scene could be drawn on the editor's timeline at one time and actually play at another. Nothing failed; the timeline just showed the clip in the wrong place, and a clip pushed past the end of the composition disappeared from it entirely. The cause was that the code drawing the timeline and the code playing the video each worked out the clip's start time from the same HTML attributes in their own way, and only one of them knew about the marker that says "this start time is already measured from the beginning of the whole video". Both now ask the same function. The same function also answers for the visibility pass, the audio scheduling paths, and the volume-fade probe, all of which were reading the raw attribute and so placed a nested clip's audio at the wrong moment. Also folds the duplicated media-length helper into one shared version. * refactor(core): resolve the volume probe window once per element * fix(core): resolve media starts through the pass-scoped timing resolver After rebasing onto the resolver-reuse change, the shared media start resolver was building a fresh start-time resolver per call, which is the per-element construction that change removed. Route it through the scoped resolver so one pass shares one set of caches.
1 parent ac27b15 commit 5973273

7 files changed

Lines changed: 230 additions & 67 deletions

File tree

‎packages/core/src/runtime/init.test.ts‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
33
import { readFileSync } from "node:fs";
44
import { initSandboxRuntimeModular } from "./init";
5+
import { collectRuntimeTimelinePayload } from "./timeline";
56
import { TYPEGPU_PRESENT_HEARTBEAT_MS } from "./adapters/typegpu";
67
import { WebAudioTransport } from "./webAudioTransport";
78
import type { RuntimeTimelineLike } from "./types";
@@ -2199,6 +2200,83 @@ describe("initSandboxRuntimeModular", () => {
21992200
expect(pipVideo.style.visibility).toBe("hidden");
22002201
});
22012202

2203+
// The clip manifest the studio timeline draws and the runtime that actually
2204+
// plays the media must resolve a media element's absolute start through the
2205+
// same function. When they disagree the editor is a lie: the clip is drawn at
2206+
// one time and plays at another, and nothing fails.
2207+
//
2208+
// Both cases below are read from the SAME DOM the runtime just initialised,
2209+
// so the expected value is whatever playback uses, never a hardcoded number.
2210+
it("reports the same media start in the clip manifest as the runtime plays at", () => {
2211+
const root = document.createElement("div");
2212+
root.setAttribute("data-composition-id", "main");
2213+
root.setAttribute("data-root", "true");
2214+
root.setAttribute("data-start", "0");
2215+
document.body.appendChild(root);
2216+
2217+
const host = document.createElement("div");
2218+
host.setAttribute("data-composition-id", "scene-pip");
2219+
host.setAttribute("data-composition-file", "compositions/pip.html");
2220+
host.setAttribute("data-start", "45.40");
2221+
host.setAttribute("data-duration", "7.06");
2222+
root.appendChild(host);
2223+
2224+
// Legacy root-global authoring: data-start is already absolute, so the host
2225+
// offset must NOT be added on top of it.
2226+
const pipVideo = document.createElement("video");
2227+
pipVideo.id = "pip";
2228+
pipVideo.setAttribute("data-start", "45.40");
2229+
pipVideo.setAttribute("data-hf-media-start-basis", "global");
2230+
pipVideo.setAttribute("data-duration", "7.06");
2231+
host.appendChild(pipVideo);
2232+
2233+
window.__timelines = {
2234+
main: createMockTimeline(60),
2235+
"scene-pip": createMockTimeline(7.06),
2236+
};
2237+
initSandboxRuntimeModular();
2238+
2239+
const runtimeStart = window.__hfResolveMediaStartSeconds?.(pipVideo);
2240+
const manifestClip = collectRuntimeTimelinePayload({ canonicalFps: 30 }).clips.find(
2241+
(clip) => clip.id === "pip",
2242+
);
2243+
expect(runtimeStart).toBeCloseTo(45.4);
2244+
expect(manifestClip?.start).toBeCloseTo(runtimeStart!);
2245+
});
2246+
2247+
it("keeps a composition-local media clip in the manifest at the time it plays", () => {
2248+
const root = document.createElement("div");
2249+
root.setAttribute("data-composition-id", "main");
2250+
root.setAttribute("data-root", "true");
2251+
root.setAttribute("data-start", "0");
2252+
document.body.appendChild(root);
2253+
2254+
// No data-duration and no registered timeline anywhere, so the media window
2255+
// IS the composition's duration — which is what the second, attribute-only
2256+
// start derivation used to get wrong.
2257+
const host = document.createElement("div");
2258+
host.setAttribute("data-composition-id", "scene-a");
2259+
host.setAttribute("data-start", "10");
2260+
root.appendChild(host);
2261+
2262+
const nested = document.createElement("video");
2263+
nested.id = "nested";
2264+
nested.setAttribute("data-start", "2");
2265+
nested.setAttribute("data-duration", "3");
2266+
host.appendChild(nested);
2267+
2268+
window.__timelines = {};
2269+
initSandboxRuntimeModular();
2270+
2271+
const runtimeStart = window.__hfResolveMediaStartSeconds?.(nested);
2272+
const manifestClip = collectRuntimeTimelinePayload({ canonicalFps: 30 }).clips.find(
2273+
(clip) => clip.id === "nested",
2274+
);
2275+
expect(runtimeStart).toBeCloseTo(12);
2276+
expect(manifestClip?.start).toBeCloseTo(runtimeStart!);
2277+
expect(manifestClip?.duration).toBeCloseTo(3);
2278+
});
2279+
22022280
it("shows auto-injected video at host time, not at t=0", () => {
22032281
const root = document.createElement("div");
22042282
root.setAttribute("data-composition-id", "main");

‎packages/core/src/runtime/init.ts‎

Lines changed: 11 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,8 @@ import type { PlayerAPI } from "../core.types";
6565
import { swallow } from "./diagnostics";
6666
import { shouldAttemptPeriodicTimelineBind } from "./timelineRebindPolicy";
6767
import { installStudioCustomEase } from "./customEase";
68-
import { parseNumeric } from "./startExpression";
69-
import { parseStrictFiniteTimingNumber } from "./playbackRate";
70-
import {
71-
MEDIA_START_BASIS_ATTR,
72-
resolveAbsoluteMediaStartSeconds as resolveAuthoredMediaStartSeconds,
73-
} from "../mediaTiming";
68+
import { parseStrictFiniteTimingNumber, resolveMediaElementDurationSeconds } from "./playbackRate";
69+
import { MEDIA_START_BASIS_ATTR } from "../mediaTiming";
7470
import {
7571
clearRuntimeData,
7672
setRuntimeData,
@@ -712,24 +708,11 @@ export function initSandboxRuntimeModular(): void {
712708
return { compositionRoot, inheritedStart, inheritedDuration };
713709
};
714710

715-
const resolveAbsoluteMediaStartSeconds = (element: Element): number => {
716-
const context = resolveMediaCompositionContext(element);
717-
const inheritedStart = context.inheritedStart ?? 0;
718-
const authoredStart = parseNumeric(element.getAttribute("data-start"));
719-
if (
720-
element.hasAttribute("data-hf-auto-start") ||
721-
authoredStart == null ||
722-
inheritedStart <= 0
723-
) {
724-
return resolveStartForElement(element, inheritedStart);
725-
}
726-
727-
return resolveAuthoredMediaStartSeconds({
728-
authoredStart,
729-
hostStart: inheritedStart,
730-
basis: element.getAttribute(MEDIA_START_BASIS_ATTR),
731-
});
732-
};
711+
// Single owner: `createRuntimeStartTimeResolver` (startResolver.ts). The clip
712+
// manifest resolves media starts through the same method, so what the studio
713+
// draws and what the transport plays cannot drift apart.
714+
const resolveAbsoluteMediaStartSeconds = (element: Element): number =>
715+
timingResolverFor(true).resolveMediaStartForElement(element);
733716

734717
window.__hfResolveMediaStartSeconds = resolveAbsoluteMediaStartSeconds;
735718
runtimeCleanupCallbacks.push(() => {
@@ -832,17 +815,6 @@ export function initSandboxRuntimeModular(): void {
832815
};
833816
};
834817

835-
const resolveMediaElementDurationSeconds = (node: HTMLMediaElement): number | null => {
836-
const declaredDuration = parseStrictFiniteTimingNumber(node.getAttribute("data-duration"));
837-
if (declaredDuration != null && declaredDuration > 0) {
838-
return declaredDuration;
839-
}
840-
if (Number.isFinite(node.duration)) {
841-
return resolveNaturalMediaTimelineDuration(node, node.duration);
842-
}
843-
return null;
844-
};
845-
846818
// Scope 3 of 3 (see `withTimingResolver`). Every media element resolves its
847819
// composition ancestry here, and this runs on every transport tick via
848820
// `getSafeTimelineDurationSeconds`, so it is the heaviest consumer of the
@@ -3282,7 +3254,7 @@ export function initSandboxRuntimeModular(): void {
32823254
for (const rawEl of audioEls) {
32833255
if (!(rawEl instanceof HTMLMediaElement) || !rawEl.isConnected) continue;
32843256
if (isSilencedByHidden(rawEl)) continue;
3285-
const start = Number.parseFloat(rawEl.dataset.start ?? "");
3257+
const start = resolveAbsoluteMediaStartSeconds(rawEl);
32863258
const durAttr = parseStrictFiniteTimingNumber(rawEl.dataset.duration);
32873259
const end = durAttr != null && durAttr > 0 ? start + durAttr : Infinity;
32883260
const mediaStart = readElementPlaybackStart(rawEl);
@@ -3373,7 +3345,8 @@ export function initSandboxRuntimeModular(): void {
33733345
for (const el of mediaEls) {
33743346
if (!(el instanceof HTMLMediaElement)) continue;
33753347
if (!el.isConnected) continue;
3376-
const start = Number.parseFloat(el.dataset.start ?? "");
3348+
if (!el.hasAttribute("data-start")) continue;
3349+
const start = resolveAbsoluteMediaStartSeconds(el);
33773350
if (!Number.isFinite(start)) continue;
33783351
const durAttr = parseStrictFiniteTimingNumber(el.dataset.duration);
33793352
const end = durAttr != null && durAttr > 0 ? start + durAttr : Infinity;
@@ -3403,7 +3376,7 @@ export function initSandboxRuntimeModular(): void {
34033376
for (const rawEl of audioEls) {
34043377
if (!(rawEl instanceof HTMLMediaElement) || !rawEl.isConnected) continue;
34053378
if (isSilencedByHidden(rawEl)) continue;
3406-
const compStart = Number.parseFloat(rawEl.dataset.start ?? "");
3379+
const compStart = resolveAbsoluteMediaStartSeconds(rawEl);
34073380
if (!Number.isFinite(compStart)) continue;
34083381
const mediaStart = readElementPlaybackStart(rawEl);
34093382
const volumeAttr = Number.parseFloat(rawEl.dataset.volume ?? "");

‎packages/core/src/runtime/mediaVolumeEnvelope.test.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,41 @@ describe("probeAndCacheElementVolume", () => {
201201
expect(interpolateVolumeGain(envelope, 0.5)).toBeCloseTo(1, 5);
202202
expect(interpolateVolumeGain(envelope, 1)).toBeCloseTo(1, 5);
203203
});
204+
it("uses the clip's absolute start, not its composition-local data-start", () => {
205+
// Same fade as above, but the clip lives in a host composition that begins
206+
// at t=2, so its `data-start="1"` means timeline t=3. Reading the attribute
207+
// directly probed [1,2] — a window the clip is not even on screen for — and
208+
// rebased the envelope 2s early.
209+
const host = document.createElement("div");
210+
host.setAttribute("data-composition-id", "scene-a");
211+
host.dataset.start = "2";
212+
document.body.append(host);
213+
const audio = document.createElement("audio");
214+
audio.dataset.start = "1";
215+
audio.dataset.duration = "1";
216+
audio.dataset.volume = "1";
217+
host.append(audio);
218+
219+
const timeline = {
220+
totalTime(next?: number) {
221+
if (next !== undefined) {
222+
// 0.05s linear fade-in at the clip's real start (timeline t=3).
223+
audio.volume = Math.max(0, Math.min(1, (next - 3) / 0.05));
224+
}
225+
return 0;
226+
},
227+
};
228+
const cache = new WeakMap<HTMLMediaElement, { time: number; volume: number }[]>();
229+
230+
probeAndCacheElementVolume(audio, timeline, 4, cache);
231+
232+
const envelope = cache.get(audio);
233+
if (!envelope) throw new Error("Expected a cached envelope");
234+
expect(interpolateVolumeGain(envelope, 0)).toBeCloseTo(0, 5);
235+
expect(interpolateVolumeGain(envelope, 0.05)).toBeCloseTo(1, 5);
236+
expect(interpolateVolumeGain(envelope, 1)).toBeCloseTo(1, 5);
237+
});
238+
204239
it("keeps a fade that starts from an above-unity authored gain", () => {
205240
const audio = document.createElement("audio");
206241
audio.dataset.start = "0";

‎packages/core/src/runtime/mediaVolumeEnvelope.ts‎

Lines changed: 39 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import type { RuntimeTimelineLike } from "./types";
22
import { clampAudioGain, withUnclampedVolume } from "../audioGain.js";
33
import { parseStrictFiniteTimingNumber } from "./playbackRate";
4+
import { createRuntimeStartTimeResolver } from "./startResolver";
45

56
/**
67
* Shared volume-automation utilities used by both the renderer (offline PCM
@@ -102,11 +103,26 @@ function parseVolumeNumber(value: string | undefined): number | undefined {
102103
return Number.isFinite(parsed) ? parsed : undefined;
103104
}
104105

106+
interface ProbeWindow {
107+
start: number;
108+
end: number;
109+
staticVolume: number;
110+
}
111+
105112
function resolveVolumeProbeWindow(
106113
el: HTMLAudioElement | HTMLVideoElement,
107114
compositionDuration: number,
108-
): { start: number; end: number; staticVolume: number } {
109-
const start = parseStrictFiniteTimingNumber(el.dataset.start) ?? 0;
115+
): ProbeWindow {
116+
// Probe samples are stamped with ROOT-timeline seek times, and
117+
// `normaliseEnvelope` rebases them by this start — so it has to be the same
118+
// absolute start the transport plays the clip at. Reading `data-start`
119+
// directly gave a composition-local value, which put a nested clip's whole
120+
// envelope at the wrong origin.
121+
const start = createRuntimeStartTimeResolver({
122+
timelineRegistry: (window as Window & { __timelines?: Record<string, RuntimeTimelineLike> })
123+
.__timelines,
124+
includeAuthoredTimingAttrs: true,
125+
}).resolveMediaStartForElement(el);
110126
const endAttr = parseStrictFiniteTimingNumber(el.dataset.end) ?? undefined;
111127
const durAttr = parseStrictFiniteTimingNumber(el.dataset.duration) ?? undefined;
112128
let end = compositionDuration;
@@ -135,8 +151,25 @@ export function probeElementVolumeKeyframes(
135151
compositionDuration: number,
136152
sampleFps: number,
137153
): VolumeKeyframe[] | null {
138-
const { start, end, staticVolume } = resolveVolumeProbeWindow(el, compositionDuration);
154+
return probeKeyframesInWindow(
155+
el,
156+
seekTimeline,
157+
compositionDuration,
158+
sampleFps,
159+
resolveVolumeProbeWindow(el, compositionDuration),
160+
);
161+
}
139162

163+
/** Sampling half of the probe, given an already-resolved window. Split out so
164+
* `probeAndCacheElementVolume` resolves that window ONCE and reuses it for the
165+
* envelope rebase, instead of deriving the same start twice per element. */
166+
function probeKeyframesInWindow(
167+
el: HTMLAudioElement | HTMLVideoElement,
168+
seekTimeline: (t: number) => void,
169+
compositionDuration: number,
170+
sampleFps: number,
171+
{ start, end, staticVolume }: ProbeWindow,
172+
): VolumeKeyframe[] | null {
140173
const step = 1 / Math.min(60, Math.max(1, sampleFps));
141174
const sampleStart = Math.max(0, start);
142175
const sampleEnd = Math.min(compositionDuration, end);
@@ -225,11 +258,11 @@ export function probeAndCacheElementVolume(
225258
: typeof timeline.seek === "function"
226259
? Number(timeline.seek())
227260
: 0;
228-
const keyframes = probeElementVolumeKeyframes(mediaEl, seekFn, compositionDuration, 60);
261+
const probeWindow = resolveVolumeProbeWindow(mediaEl, compositionDuration);
262+
const keyframes = probeKeyframesInWindow(mediaEl, seekFn, compositionDuration, 60, probeWindow);
229263
if (Number.isFinite(originalTime)) seekFn(originalTime);
230264
if (keyframes) {
231-
const { start, staticVolume } = resolveVolumeProbeWindow(mediaEl, compositionDuration);
232-
const envelope = normaliseEnvelope(keyframes, start, staticVolume);
265+
const envelope = normaliseEnvelope(keyframes, probeWindow.start, probeWindow.staticVolume);
233266
if (envelope.length > 0) cache.set(mediaEl, envelope);
234267
}
235268
}

‎packages/core/src/runtime/playbackRate.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,23 @@ export function resolveNaturalMediaTimelineDuration(
3939
);
4040
}
4141

42+
/**
43+
* How long a media element occupies the timeline: an explicit `data-duration`
44+
* trim if authored, otherwise the natural source length adjusted for playback
45+
* start and rate. `null` when the source has not reported a duration yet.
46+
*
47+
* Single owner for the media-window scan run by BOTH the runtime's duration
48+
* floor and the clip manifest.
49+
*/
50+
export function resolveMediaElementDurationSeconds(
51+
el: Pick<Element, "getAttribute"> & { duration: number },
52+
): number | null {
53+
const declaredDuration = parseStrictFiniteTimingNumber(el.getAttribute("data-duration"));
54+
if (declaredDuration != null && declaredDuration > 0) return declaredDuration;
55+
if (Number.isFinite(el.duration)) return resolveNaturalMediaTimelineDuration(el, el.duration);
56+
return null;
57+
}
58+
4259
export function resolveNaturalMediaTimelineDurationFromValues(
4360
sourceDuration: number,
4461
mediaStart: number,

‎packages/core/src/runtime/startResolver.ts‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,16 @@
11
import type { RuntimeTimelineLike } from "./types";
22
import { swallow } from "./diagnostics";
33
import { resolveAuthoredTimingWindow } from "./authoredTiming";
4-
import { readElementPlaybackRate } from "./media";
5-
import { readMediaStart } from "./playbackRate";
4+
// Straight from playbackRate, not through media.ts's re-export: media.ts
5+
// imports mediaVolumeEnvelope, which needs this resolver, and the round trip
6+
// would be an import cycle.
7+
import {
8+
parseStrictFiniteTimingNumber,
9+
readElementPlaybackRate,
10+
readMediaStart,
11+
} from "./playbackRate";
612
import { parseStartExpression } from "./startExpression";
13+
import { MEDIA_START_BASIS_ATTR, resolveAbsoluteMediaStartSeconds } from "../mediaTiming";
714

815
export function createRuntimeStartTimeResolver(params: {
916
timelineRegistry?: Record<string, RuntimeTimelineLike | undefined>;
@@ -18,6 +25,7 @@ export function createRuntimeStartTimeResolver(params: {
1825
}): {
1926
resolveStartForElement: (element: Element, fallback?: number) => number;
2027
resolveDurationForElement: (element: Element) => number | null;
28+
resolveMediaStartForElement: (element: Element) => number;
2129
} {
2230
const timelineRegistry = params.timelineRegistry ?? {};
2331
const includeAuthoredTimingAttrs = params.includeAuthoredTimingAttrs ?? false;
@@ -175,10 +183,39 @@ export function createRuntimeStartTimeResolver(params: {
175183
}
176184
};
177185

186+
/**
187+
* The ONE owner of "when does this media element start on the root timeline".
188+
*
189+
* A media element is not a plain timed clip: `data-hf-media-start-basis`
190+
* decides whether its `data-start` is composition-local (the default, so the
191+
* host offset is added) or a legacy root-global timestamp (already absolute,
192+
* so adding the host offset double-counts it). Anything that derives a media
193+
* start from attributes — the clip manifest, the visibility pass, the media
194+
* cache, WebAudio scheduling — must come through here, or the timeline the
195+
* editor draws stops matching the timeline that plays.
196+
*/
197+
const resolveMediaStartForElement = (element: Element): number => {
198+
const compositionRoot = element.closest("[data-composition-id]");
199+
const hostStart = compositionRoot ? resolveStartForElementInternal(compositionRoot, 0) : 0;
200+
const authoredStart = parseStrictFiniteTimingNumber(element.getAttribute("data-start"));
201+
// No literal start (absent, or a `data-start="intro + 2"` reference), an
202+
// auto-injected start, or a host at t=0 — nothing for the basis to
203+
// disambiguate, so the ordinary start resolution is already correct.
204+
if (element.hasAttribute("data-hf-auto-start") || authoredStart == null || hostStart <= 0) {
205+
return resolveStartForElementInternal(element, hostStart);
206+
}
207+
return resolveAbsoluteMediaStartSeconds({
208+
authoredStart,
209+
hostStart,
210+
basis: element.getAttribute(MEDIA_START_BASIS_ATTR),
211+
});
212+
};
213+
178214
return {
179215
resolveStartForElement: (element: Element, fallback = 0) =>
180216
resolveStartForElementInternal(element, Math.max(0, fallback)),
181217
resolveDurationForElement: (element: Element) => resolveDurationForElement(element),
218+
resolveMediaStartForElement,
182219
};
183220
}
184221

0 commit comments

Comments
 (0)