Skip to content

Commit 2fc5a37

Browse files
fix(core): a video's sound plays at its boosted volume in the preview (#4746)
* fix(core): a video's sound plays at its boosted volume in the preview A <video> with data-volume above 1 now plays through the same Web Audio gain as <audio>, so the preview matches the export. Videos at or below 1 keep their native output. * style(core): keep the runtime's comment share at its base * fix(core): a boosted video hidden after its volume drops goes silent
1 parent efd3c51 commit 2fc5a37

2 files changed

Lines changed: 104 additions & 15 deletions

File tree

‎packages/core/src/runtime/init.test.ts‎

Lines changed: 83 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4945,7 +4945,11 @@ describe("initSandboxRuntimeModular", () => {
49454945
await Promise.resolve();
49464946
}
49474947

4948-
function mountAudio(src: string, attrs: Record<string, string> = {}) {
4948+
function mountAudio(
4949+
src: string,
4950+
attrs: Record<string, string> = {},
4951+
tag: "audio" | "video" = "audio",
4952+
) {
49494953
const root = document.createElement("div");
49504954
root.setAttribute("data-composition-id", "main");
49514955
root.setAttribute("data-root", "true");
@@ -4955,7 +4959,7 @@ describe("initSandboxRuntimeModular", () => {
49554959
root.setAttribute("data-height", "1080");
49564960
document.body.appendChild(root);
49574961

4958-
const audio = document.createElement("audio");
4962+
const audio = document.createElement(tag);
49594963
audio.setAttribute("data-start", "0");
49604964
audio.setAttribute("data-duration", "10");
49614965
audio.setAttribute("src", src);
@@ -5042,7 +5046,7 @@ describe("initSandboxRuntimeModular", () => {
50425046
expect(String(line?.[0])).toContain("fx-chain");
50435047
});
50445048

5045-
it("says nothing about a cross-origin <video>, which never routes through Web Audio", () => {
5049+
it("says nothing about a cross-origin <video> at unity, which never routes through Web Audio", () => {
50465050
const root = document.createElement("div");
50475051
root.setAttribute("data-composition-id", "main");
50485052
root.setAttribute("data-root", "true");
@@ -5128,6 +5132,82 @@ describe("initSandboxRuntimeModular", () => {
51285132
});
51295133
});
51305134

5135+
// `el.volume` stops at 1, so a video's authored boost can only be carried by a Web Audio gain.
5136+
describe("a video's sound above unity", () => {
5137+
const ctx = useMockAudioContext();
5138+
5139+
it("plays a video at data-volume 1.5 through a Web Audio gain of 1.5", async () => {
5140+
const video = mountAudio("/assets/broll.mp4", { "data-volume": "1.5" }, "video");
5141+
const captureSpy = vi.spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback");
5142+
5143+
await startPlayback();
5144+
const scheduled = await captureSpy.mock.results[0]?.value;
5145+
5146+
expect(captureSpy.mock.calls[0]?.[0]).toBe(video);
5147+
expect(ctx.mediaElementSources).toBe(1);
5148+
expect(scheduled?.gainNode.gain.value).toBe(1.5);
5149+
expect(video.volume).toBe(1);
5150+
expect(video.muted).toBe(false);
5151+
});
5152+
5153+
it("leaves a video at unity on its native output", async () => {
5154+
mountAudio("/assets/broll.mp4", { "data-volume": "1" }, "video");
5155+
const captureSpy = vi.spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback");
5156+
5157+
await startPlayback();
5158+
5159+
expect(captureSpy).not.toHaveBeenCalled();
5160+
expect(ctx.mediaElementSources).toBe(0);
5161+
});
5162+
5163+
it("silences a captured video hidden after its volume drops to 1", async () => {
5164+
const raf = createManualRaf();
5165+
vi.spyOn(performance, "now").mockImplementation(() => raf.now());
5166+
window.requestAnimationFrame =
5167+
raf.requestAnimationFrame as typeof window.requestAnimationFrame;
5168+
window.cancelAnimationFrame = raf.cancelAnimationFrame as typeof window.cancelAnimationFrame;
5169+
const video = mountAudio("/assets/broll.mp4", { "data-volume": "1.5" }, "video");
5170+
Object.defineProperty(video, "paused", { value: false, configurable: true });
5171+
Object.defineProperty(video, "readyState", { value: 4, configurable: true });
5172+
const captureSpy = vi.spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback");
5173+
const step = async () => {
5174+
for (let frame = 0; frame < 5; frame++) {
5175+
ctx.time += 1 / 60;
5176+
raf.step(1000 / 60);
5177+
await Promise.resolve();
5178+
}
5179+
};
5180+
5181+
await startPlayback();
5182+
await captureSpy.mock.results[0]?.value;
5183+
const transport = captureSpy.mock.contexts[0] as WebAudioTransport;
5184+
await step();
5185+
video.setAttribute("data-volume", "1");
5186+
await step();
5187+
video.setAttribute("data-hidden", "");
5188+
await step();
5189+
5190+
expect(transport.routesElement(video)).toBe(false);
5191+
expect(video.volume).toBe(0);
5192+
});
5193+
5194+
it("keeps a cross-origin boosted video native and never decodes it", async () => {
5195+
const video = mountAudio(
5196+
"https://cdn.example.com/broll.mp4",
5197+
{ "data-volume": "1.5" },
5198+
"video",
5199+
);
5200+
vi.spyOn(console, "info").mockImplementation(() => {});
5201+
const decodeSpy = vi.spyOn(WebAudioTransport.prototype, "decodeAudioElement");
5202+
5203+
await startPlayback();
5204+
5205+
expect(ctx.mediaElementSources).toBe(0);
5206+
expect(decodeSpy).not.toHaveBeenCalled();
5207+
expect(video.muted).toBe(false);
5208+
});
5209+
});
5210+
51315211
describe("a voiceover routed through Web Audio", () => {
51325212
const ctx = useMockAudioContext();
51335213

‎packages/core/src/runtime/init.ts‎

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,13 @@ function pageAnimationsForOnePass(): () => Animation[] {
329329
return () => (list ??= document.getAnimations());
330330
}
331331

332+
// A `<video>` joins only for a gain `el.volume` cannot express: capture is a one-way door.
333+
const joinsWebAudio = (el: Element): el is HTMLMediaElement =>
334+
isAudioElement(el) || (isVideoElement(el) && Number.parseFloat(el.dataset.volume ?? "") > 1);
335+
const WEB_AUDIO_MEDIA = "audio[data-start], video[data-start]";
336+
const webAudioMediaIn = (root: ParentNode): HTMLMediaElement[] =>
337+
Array.from(root.querySelectorAll(WEB_AUDIO_MEDIA)).filter(joinsWebAudio);
338+
332339
export function initSandboxRuntimeModular(): void {
333340
const state = createRuntimeState();
334341
authoredMediaObserver?.disconnect();
@@ -2379,12 +2386,10 @@ export function initSandboxRuntimeModular(): void {
23792386
}
23802387
};
23812388

2382-
// Only `<audio>` reaches `createMediaElementSource` (see
2383-
// `scheduleWebAudioForActiveClips`, which queries `audio[data-start]`), so a
2384-
// cross-origin `<video>` is not affected and must not be reported as if it
2385-
// were.
2389+
// Only media `joinsWebAudio` admits when a play is scheduled reaches
2390+
// `createMediaElementSource`, so no other cross-origin media may be reported as if it did.
23862391
const reportWebAudioRoute = (mediaEl: HTMLMediaElement) => {
2387-
if (!isAudioElement(mediaEl)) return;
2392+
if (!joinsWebAudio(mediaEl)) return;
23882393
// Before resource selection settles, the verdict is built from `<source>`
23892394
// children the browser might still pass over — good enough for the
23902395
// schedule path's conservative withhold, not good enough to put in front
@@ -2571,8 +2576,12 @@ export function initSandboxRuntimeModular(): void {
25712576
// started a second buffer source for every in-window clip on top of the ones
25722577
// still sounding: the whole mix audibly doubled, slightly out of phase.
25732578
let hiddenAudioDirty = false;
2579+
const affectsAudio = (el: Element): boolean =>
2580+
isMediaElement(el) &&
2581+
el.hasAttribute("data-start") &&
2582+
(joinsWebAudio(el) || webAudio.routesElement(el));
25742583
const nodeAffectsAudio = (node: HTMLElement): boolean =>
2575-
node.matches("audio[data-start]") || node.querySelector("audio[data-start]") !== null;
2584+
affectsAudio(node) || Array.from(node.querySelectorAll(WEB_AUDIO_MEDIA)).some(affectsAudio);
25762585

25772586
// An `<hf-audio-group>` carries no `data-start`, so it is never among
25782587
// `visibilityNodes` above — group mute needs its own small diff pass.
@@ -2796,8 +2805,8 @@ export function initSandboxRuntimeModular(): void {
27962805
const groupNeedsCapture = syncAudioGroupMute(currentTime);
27972806
if ((hiddenAudioDirty || groupNeedsCapture) && clock.isPlaying()) {
27982807
webAudio.stopAll();
2799-
for (const el of document.querySelectorAll("audio[data-start]")) {
2800-
if (isMediaElement(el) && isSilencedByHidden(el)) el.volume = 0;
2808+
for (const el of webAudioMediaIn(document)) {
2809+
if (isSilencedByHidden(el)) el.volume = 0;
28012810
}
28022811
scheduleWebAudioForActiveClips();
28032812
}
@@ -4765,9 +4774,7 @@ export function initSandboxRuntimeModular(): void {
47654774
const scheduleWebAudioForActiveClips = () => {
47664775
if (state.nativeMediaSyncDisabled || state.webAudioMediaDisabled) return;
47674776
const gen = webAudio.startGeneration();
4768-
const audioEls = document.querySelectorAll("audio[data-start]");
4769-
for (const rawEl of audioEls) {
4770-
if (!isMediaElement(rawEl) || !rawEl.isConnected) continue;
4777+
for (const rawEl of webAudioMediaIn(document)) {
47714778
if (isSilencedByHidden(rawEl)) continue;
47724779
const compStart = resolveAbsoluteMediaStartSeconds(rawEl);
47734780
if (!Number.isFinite(compStart)) continue;
@@ -4814,7 +4821,9 @@ export function initSandboxRuntimeModular(): void {
48144821
: Promise.resolve(null);
48154822
void capture.then((scheduled) => {
48164823
const replacedByNewerPass = gen !== webAudio.currentGeneration();
4817-
if (scheduled || !clock.isPlaying() || replacedByNewerPass) return;
4824+
// A video's picture must keep playing from the element, so it has no decode fallback.
4825+
if (scheduled || !isAudioElement(rawEl) || !clock.isPlaying() || replacedByNewerPass)
4826+
return;
48184827
const effectiveRate = state.playbackRate * readElementPlaybackRate(rawEl);
48194828
// Deliberately the FX/automation pair and NOT
48204829
// `nativeUnexpressibleProcessing()`, which this route's diagnostic uses.

0 commit comments

Comments
 (0)