Skip to content

Commit 2ca4930

Browse files
fix(producer): follow variable audio duration (#3647)
1 parent 78a28b8 commit 2ca4930

5 files changed

Lines changed: 182 additions & 22 deletions

File tree

‎packages/producer/src/services/htmlCompiler.naturalDuration.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,21 @@ describe("compileForRender natural media duration parity", () => {
7979
}
8080
});
8181

82+
it("marks only variable-bound media whose natural duration was inferred", async () => {
83+
const { document } = await compile(`
84+
<audio id="inferred" src="ten-seconds.wav" data-var-src="track"></audio>
85+
<audio id="authored" src="ten-seconds.wav" data-var-src="track" data-duration="4"></audio>
86+
<audio id="plain" src="ten-seconds.wav"></audio>`);
87+
88+
expect(document.getElementById("inferred")?.hasAttribute("data-hf-inferred-duration")).toBe(
89+
true,
90+
);
91+
expect(document.getElementById("authored")?.hasAttribute("data-hf-inferred-duration")).toBe(
92+
false,
93+
);
94+
expect(document.getElementById("plain")?.hasAttribute("data-hf-inferred-duration")).toBe(false);
95+
});
96+
8297
it("uses shared playback-start precedence and fallback semantics", async () => {
8398
const cases = [
8499
["precedence", 'data-playback-start="2" data-media-start="7"', 8],

‎packages/producer/src/services/htmlCompiler.test.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,17 @@ describe("discoverMediaFromBrowser", () => {
3434
html: string,
3535
currentSrcById: Record<string, string>,
3636
serializeCallback = false,
37+
intrinsicDurationById: Record<string, number> = {},
3738
) {
3839
const { document } = parseHTML(html);
3940
for (const [id, currentSrc] of Object.entries(currentSrcById)) {
4041
const element = document.getElementById(id);
4142
if (element) Object.defineProperty(element, "currentSrc", { value: currentSrc });
4243
}
44+
for (const [id, duration] of Object.entries(intrinsicDurationById)) {
45+
const element = document.getElementById(id);
46+
if (element) Object.defineProperty(element, "duration", { value: duration });
47+
}
4348
const previousDocument = Reflect.get(globalThis, "document");
4449
Reflect.set(globalThis, "document", document);
4550
try {
@@ -82,6 +87,21 @@ describe("discoverMediaFromBrowser", () => {
8287
expect(media[0]).toMatchObject({ id: "hf-img-1", tagName: "image" });
8388
});
8489

90+
it("uses intrinsic duration only for variable media with an inferred duration", async () => {
91+
const media = await discover(
92+
`<audio id="inferred" src="fallback.wav" data-start="0" data-duration="3" data-end="3" data-var-src="track" data-hf-inferred-duration></audio>
93+
<audio id="authored" src="fallback.wav" data-start="0" data-duration="3" data-end="3" data-var-src="track"></audio>`,
94+
{ inferred: "selected.wav", authored: "selected.wav" },
95+
false,
96+
{ inferred: 6.530612, authored: 6.530612 },
97+
);
98+
99+
expect(media.find((item) => item.id === "inferred")?.duration).toBe(6.530612);
100+
expect(media.find((item) => item.id === "inferred")?.durationInferred).toBe(true);
101+
expect(media.find((item) => item.id === "authored")?.duration).toBe(3);
102+
expect(media.find((item) => item.id === "authored")?.durationInferred).toBe(false);
103+
});
104+
85105
it("discovers the owning image for a variable-bound picture source", async () => {
86106
const media = await discover(
87107
`<picture>

‎packages/producer/src/services/htmlCompiler.ts‎

Lines changed: 47 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,8 @@ export interface CompiledComposition {
104104
hasAncestorBackgroundImage: boolean;
105105
}
106106

107+
const INFERRED_MEDIA_DURATION_ATTR = "data-hf-inferred-duration";
108+
107109
/** Adapts linkedom's `parseHTML` to the `checkSubCompositionUsability` contract. */
108110
function parseSubCompHtmlForValidity(html: string): ParsableDocumentLike {
109111
return parseHTML(html).document as unknown as ParsableDocumentLike;
@@ -534,6 +536,23 @@ async function resolveMediaDuration(
534536
* Compile a single HTML file: static pass + ffprobe for unresolved media.
535537
* Returns compiled HTML and any unresolved composition elements that need browser resolution.
536538
*/
539+
function markInferredVariableMediaDurations(
540+
html: string,
541+
unresolvedMedia: readonly UnresolvedElement[],
542+
): string {
543+
const unresolvedIds = new Set(unresolvedMedia.map((element) => element.id));
544+
if (unresolvedIds.size === 0) return html;
545+
546+
const { document } = parseHTML(html);
547+
let changed = false;
548+
for (const element of document.querySelectorAll("video[data-var-src], audio[data-var-src]")) {
549+
if (!unresolvedIds.has(element.id)) continue;
550+
element.setAttribute(INFERRED_MEDIA_DURATION_ATTR, "");
551+
changed = true;
552+
}
553+
return changed ? document.toString() : html;
554+
}
555+
537556
async function compileHtmlFile(
538557
html: string,
539558
baseDir: string,
@@ -567,8 +586,9 @@ async function compileHtmlFile(
567586
(r): r is ResolvedDuration => r.duration != null && Number.isFinite(r.duration),
568587
);
569588

589+
const markedStaticHtml = markInferredVariableMediaDurations(staticCompiled, mediaUnresolved);
570590
let compiledHtml =
571-
resolutions.length > 0 ? injectDurations(staticCompiled, resolutions) : staticCompiled;
591+
resolutions.length > 0 ? injectDurations(markedStaticHtml, resolutions) : markedStaticHtml;
572592

573593
// Phase 2: Bound authored audio to playable source (parallel ffprobe).
574594
// Explicit video slots may outlive their source and hold the final frame.
@@ -2101,6 +2121,8 @@ export interface BrowserMediaElement {
21012121
start: number;
21022122
end: number;
21032123
duration: number;
2124+
/** True when compilation inferred duration from the fallback source. */
2125+
durationInferred: boolean;
21042126
mediaStart: number;
21052127
loop: boolean;
21062128
hasAudio: boolean;
@@ -2123,6 +2145,8 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
21232145
start: number;
21242146
endRaw: string | null;
21252147
durationRaw: string | null;
2148+
intrinsicDuration: number;
2149+
durationInferred: boolean;
21262150
playbackStartRaw: string | null;
21272151
mediaStartRaw: string | null;
21282152
loop: boolean;
@@ -2169,6 +2193,10 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
21692193
const start = parseFloat(htmlEl.getAttribute("data-start") || "0");
21702194
const endRaw = htmlEl.getAttribute("data-end");
21712195
const durationRaw = htmlEl.getAttribute("data-duration");
2196+
const durationInferred = htmlEl.hasAttribute("data-hf-inferred-duration");
2197+
const intrinsicDuration = isImage
2198+
? 0
2199+
: (htmlEl as HTMLVideoElement | HTMLAudioElement).duration;
21722200
const playbackStartRaw = htmlEl.getAttribute("data-playback-start");
21732201
const mediaStartRaw = htmlEl.getAttribute("data-media-start");
21742202
const loop = htmlEl.hasAttribute("loop");
@@ -2185,6 +2213,8 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
21852213
start,
21862214
endRaw,
21872215
durationRaw,
2216+
intrinsicDuration,
2217+
durationInferred,
21882218
playbackStartRaw,
21892219
mediaStartRaw,
21902220
loop,
@@ -2197,18 +2227,23 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
21972227
return results;
21982228
});
21992229

2200-
return elements.map(({ endRaw, durationRaw, playbackStartRaw, mediaStartRaw, ...element }) => ({
2201-
...element,
2202-
end: parseStrictFiniteTimingNumber(endRaw) ?? 0,
2203-
duration: parseStrictFiniteTimingNumber(durationRaw) ?? 0,
2204-
mediaStart: readMediaStart({
2205-
getAttribute(name: string) {
2206-
if (name === "data-playback-start") return playbackStartRaw;
2207-
if (name === "data-media-start") return mediaStartRaw;
2208-
return null;
2209-
},
2230+
return elements.map(
2231+
({ endRaw, durationRaw, intrinsicDuration, playbackStartRaw, mediaStartRaw, ...element }) => ({
2232+
...element,
2233+
end: parseStrictFiniteTimingNumber(endRaw) ?? 0,
2234+
duration:
2235+
element.durationInferred && Number.isFinite(intrinsicDuration) && intrinsicDuration > 0
2236+
? intrinsicDuration
2237+
: (parseStrictFiniteTimingNumber(durationRaw) ?? 0),
2238+
mediaStart: readMediaStart({
2239+
getAttribute(name: string) {
2240+
if (name === "data-playback-start") return playbackStartRaw;
2241+
if (name === "data-media-start") return mediaStartRaw;
2242+
return null;
2243+
},
2244+
}),
22102245
}),
2211-
}));
2246+
);
22122247
}
22132248

22142249
export async function discoverAudioVolumeAutomationFromTimeline(

‎packages/producer/src/services/render/stages/probeStage.test.ts‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -444,6 +444,79 @@ describe("runProbeStage — forceScreenshot threading", () => {
444444
expect(mediaPreflightComposition).toBe(input.composition);
445445
});
446446

447+
it("uses selected intrinsic duration only when the variable-bound duration was inferred", async () => {
448+
resetRetryMocks();
449+
const media = (
450+
id: string,
451+
duration: number,
452+
durationInferred: boolean,
453+
): Record<string, unknown> => ({
454+
id,
455+
tagName: "audio",
456+
src: `${id}-selected.wav`,
457+
start: 0,
458+
end: duration,
459+
duration,
460+
durationInferred,
461+
mediaStart: 0,
462+
loop: false,
463+
hasAudio: true,
464+
volume: 1,
465+
muted: false,
466+
});
467+
browserMediaResults = [
468+
media("longer-inferred", 6.530612, true),
469+
media("longer-authored", 6.530612, false),
470+
media("shorter-inferred", 3.836939, true),
471+
];
472+
const { runProbeStage } = await import("./probeStage.js");
473+
const input = makeProbeInput({});
474+
input.composition.duration = 7;
475+
input.composition.audios.push(
476+
{
477+
id: "longer-inferred",
478+
src: "short.wav",
479+
start: 0,
480+
end: 3.836939,
481+
mediaStart: 0,
482+
layer: 0,
483+
volume: 1,
484+
type: "audio",
485+
},
486+
{
487+
id: "longer-authored",
488+
src: "short.wav",
489+
start: 0,
490+
end: 3.836939,
491+
mediaStart: 0,
492+
layer: 0,
493+
volume: 1,
494+
type: "audio",
495+
},
496+
{
497+
id: "shorter-inferred",
498+
src: "long.wav",
499+
start: 0,
500+
end: 6.530612,
501+
mediaStart: 0,
502+
layer: 0,
503+
volume: 1,
504+
type: "audio",
505+
},
506+
);
507+
input.compiled.html = `
508+
<audio id="longer-inferred" src="short.wav" data-var-src="a" data-hf-inferred-duration></audio>
509+
<audio id="longer-authored" src="short.wav" data-var-src="b" data-duration="3.836939"></audio>
510+
<audio id="shorter-inferred" src="long.wav" data-var-src="c" data-hf-inferred-duration></audio>`;
511+
input.job.config.variables = { a: "a.wav", b: "b.wav", c: "c.wav" };
512+
513+
await runProbeStage(input);
514+
515+
expect(input.composition.audios.map((audio) => audio.end)).toEqual([
516+
6.530612, 3.836939, 3.836939,
517+
]);
518+
});
519+
447520
it("passes cancellation through and closes probe-owned resources when preflight rejects", async () => {
448521
resetRetryMocks();
449522
mediaPreflightError = new Error("ASSET_MEDIA_TYPE_MISMATCH");

‎packages/producer/src/services/render/stages/probeStage.ts‎

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,17 @@ export function hasVariableBoundMedia(
176176
});
177177
}
178178

179+
function reconcileBrowserMediaEnd(
180+
existingEnd: number,
181+
projectedEnd: number,
182+
sourceChanged: boolean,
183+
durationInferred: boolean,
184+
): number {
185+
if (projectedEnd <= 0) return existingEnd;
186+
if (sourceChanged && durationInferred) return projectedEnd;
187+
return existingEnd <= 0 ? projectedEnd : Math.min(existingEnd, projectedEnd);
188+
}
189+
179190
/**
180191
* Runtime-created media does not exist when the static compiler scans the HTML.
181192
* Launch a browser probe so discoverMediaFromBrowser can reconcile it before
@@ -500,18 +511,21 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
500511
// Reconcile to browser/runtime media metadata (runtime src can differ from static HTML).
501512
const existing = composition.videos.find((v) => v.id === el.id);
502513
if (existing) {
503-
if (existing.src !== src) {
514+
const sourceChanged = existing.src !== src;
515+
if (sourceChanged) {
504516
existing.src = src;
505517
}
506518
const projectedEnd = projectBrowserEndToCompositionTimeline(
507519
existing.start,
508520
el.start,
509521
resolveBrowserMediaEnd(el.start, el.end, el.duration),
510522
);
511-
if (projectedEnd > 0) {
512-
existing.end =
513-
existing.end <= 0 ? projectedEnd : Math.min(existing.end, projectedEnd);
514-
}
523+
existing.end = reconcileBrowserMediaEnd(
524+
existing.end,
525+
projectedEnd,
526+
sourceChanged,
527+
el.durationInferred,
528+
);
515529
if (
516530
el.mediaStart > 0 &&
517531
(existing.mediaStart <= 0 ||
@@ -544,18 +558,21 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
544558
if (existingAudioIds.has(el.id)) {
545559
const existing = composition.audios.find((a) => a.id === el.id);
546560
if (existing) {
547-
if (existing.src !== src) {
561+
const sourceChanged = existing.src !== src;
562+
if (sourceChanged) {
548563
existing.src = src;
549564
}
550565
const projectedEnd = projectBrowserEndToCompositionTimeline(
551566
existing.start,
552567
el.start,
553568
resolveBrowserMediaEnd(el.start, el.end, el.duration),
554569
);
555-
if (projectedEnd > 0) {
556-
existing.end =
557-
existing.end <= 0 ? projectedEnd : Math.min(existing.end, projectedEnd);
558-
}
570+
existing.end = reconcileBrowserMediaEnd(
571+
existing.end,
572+
projectedEnd,
573+
sourceChanged,
574+
el.durationInferred,
575+
);
559576
if (
560577
el.mediaStart > 0 &&
561578
(existing.mediaStart <= 0 ||

0 commit comments

Comments
 (0)