Skip to content

Commit 0c16afd

Browse files
fix(core): Safari stops chopping audio by seeking it over and over during playback (#6854)
* fix(core): Safari stops chopping audio by seeking it over and over during playback * test(player): the WebKit playbench also counts audio seeks and frozen audio * fix(core): audio still catching up is seeked once its lag passes hard sync
1 parent a04b45a commit 0c16afd

5 files changed

Lines changed: 158 additions & 12 deletions

File tree

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
import { afterEach, describe, expect, it, vi } from "vitest";
2+
import { TransportClock } from "./clock";
3+
import { syncRuntimeMedia, type RuntimeMediaClip } from "./media";
4+
5+
// Safari's audio reports the time it was started or seeked to, ready to play, until output begins.
6+
const OUTPUT_START_MS = 150;
7+
const SEEKING_MS = 30;
8+
const TICK_MS = 16;
9+
10+
function safariAudio(now: () => number) {
11+
const el = document.createElement("audio");
12+
document.body.appendChild(el);
13+
let position = 0;
14+
let movingFromMs = Infinity;
15+
let paused = true;
16+
let rate = 1;
17+
let seeks = 0;
18+
let seekingUntilMs = -Infinity;
19+
const restart = () => (movingFromMs = now() + OUTPUT_START_MS);
20+
const advance = (dtMs: number) => {
21+
if (!paused && now() >= movingFromMs) position += (dtMs / 1000) * rate;
22+
};
23+
Object.defineProperties(el, {
24+
paused: { configurable: true, get: () => paused },
25+
seeking: { configurable: true, get: () => now() < seekingUntilMs },
26+
readyState: { configurable: true, get: () => 4 },
27+
duration: { configurable: true, get: () => 60 },
28+
played: {
29+
configurable: true,
30+
get: () => ({ length: 1, start: () => 0, end: () => position }),
31+
},
32+
playbackRate: { configurable: true, get: () => rate, set: (v: number) => (rate = v) },
33+
currentTime: {
34+
configurable: true,
35+
get: () => position,
36+
set: (v: number) => {
37+
seeks += 1;
38+
seekingUntilMs = now() + SEEKING_MS;
39+
position = v;
40+
restart();
41+
},
42+
},
43+
});
44+
el.play = vi.fn(() => {
45+
paused = false;
46+
restart();
47+
return Promise.resolve();
48+
});
49+
el.pause = vi.fn(() => void (paused = true));
50+
return { el, advance, seeks: () => seeks };
51+
}
52+
53+
/** Plays one audio clip for `seconds`; like the runtime, the playhead follows it once it is unpaused. */
54+
function playAudioClock(seconds: number) {
55+
let ms = 0;
56+
const audio = safariAudio(() => ms);
57+
const clock = new TransportClock({ nowMs: () => ms, duration: 60 });
58+
const clip: RuntimeMediaClip = {
59+
el: audio.el,
60+
start: 0,
61+
mediaStart: 0,
62+
duration: 60,
63+
end: 60,
64+
volume: null,
65+
playbackRate: 1,
66+
loop: false,
67+
sourceDuration: null,
68+
};
69+
clock.play();
70+
for (ms = TICK_MS; ms < seconds * 1000; ms += TICK_MS) {
71+
audio.advance(TICK_MS);
72+
if (!audio.el.paused)
73+
clock.attachAudioSource({ el: audio.el, compositionStart: 0, mediaStart: 0 });
74+
syncRuntimeMedia({
75+
clips: [clip],
76+
timeSeconds: clock.now(),
77+
playing: true,
78+
playbackRate: 1,
79+
getCompositionDuration: () => 60,
80+
});
81+
}
82+
return { seeks: audio.seeks(), time: clock.now(), audioTime: audio.el.currentTime };
83+
}
84+
85+
describe("an audio clock whose output starts late, as Safari's does", () => {
86+
afterEach(() => {
87+
document.body.innerHTML = "";
88+
});
89+
90+
it("plays through without seeking the audio it follows", () => {
91+
const run = playAudioClock(10);
92+
expect(run.seeks).toBe(0);
93+
expect(run.time).toBeGreaterThan(9.5);
94+
expect(Math.abs(run.time - run.audioTime)).toBeLessThan(0.1);
95+
});
96+
});

‎packages/core/src/runtime/media.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import { registerSeekCompletion } from "./adapters/seek-dispatch.js";
1616
import { cutToHostSlots, type HostSlot } from "../mediaTiming.js";
1717
import {
1818
forgetSteering,
19+
markMediaRestarted,
1920
isSteering,
2021
MEDIA_HARD_SYNC_SECONDS,
2122
MEDIA_SYNC_TOLERANCE_SECONDS,
@@ -755,6 +756,7 @@ export function syncRuntimeMedia(params: {
755756
}
756757
}
757758
holdSeekBarrierUntilVideoLands(el);
759+
markMediaRestarted(el);
758760
}
759761
playRequested.delete(el);
760762
heldAhead.delete(el);
@@ -785,6 +787,7 @@ export function syncRuntimeMedia(params: {
785787
// fire between `load()` and `addEventListener` attachment, wedging
786788
// the element waiting for a callback that never came.
787789
markPlayRequested(el);
790+
markMediaRestarted(el);
788791
// Offset read after any seek above; a loop could wrap a lap before the sample.
789792
if (isNonLoopVideo)
790793
videoStarts.set(el, {

‎packages/core/src/runtime/mediaSteer.test.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { describe, expect, it } from "vitest";
2-
import { forgetSteering, isSteering, steerMedia } from "./mediaSteer";
2+
import { forgetSteering, isSteering, markMediaRestarted, steerMedia } from "./mediaSteer";
33

44
const media = (tag: "audio" | "video") => document.createElement(tag);
55

@@ -20,6 +20,13 @@ describe("steerMedia", () => {
2020
expect(isSteering(el)).toBe(false);
2121
});
2222

23+
it("steers the lag a start left in audio, but still seeks one past hard sync", () => {
24+
const el = media("audio");
25+
markMediaRestarted(el);
26+
expect(steerMedia(el, 0.3, 1, true).rate).toBeCloseTo(1.1, 9);
27+
expect(steerMedia(el, 0.8, 1, true)).toEqual({ rate: 1, seek: true });
28+
});
29+
2330
it("keeps steering until nearly back, and forgets on request", () => {
2431
const el = media("audio");
2532
steerMedia(el, 0.06, 1, true);

‎packages/core/src/runtime/mediaSteer.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,12 @@ const AUDIO_STEER_LIMIT_SECONDS = 0.1;
1515

1616
/** Direction (+1 fast, -1 slow) an element is being steered in; absent when it plays at its authored rate. */
1717
const steering = new WeakMap<HTMLMediaElement, number>();
18+
/** Started or seeked, not yet moving in tolerance: its lag is its own output latency, which a seek only renews. */
19+
const catchingUp = new WeakSet<HTMLMediaElement>();
20+
21+
export function markMediaRestarted(el: HTMLMediaElement): void {
22+
catchingUp.add(el);
23+
}
1824

1925
/** Rate or seek for media `offset` s behind (+) or ahead (-); a lag still growing (`settled` false) never seeks. */
2026
export function steerMedia(
@@ -25,8 +31,11 @@ export function steerMedia(
2531
audioOnly = el.tagName === "AUDIO",
2632
): { rate: number; seek: boolean } {
2733
const lag = Math.abs(offset);
28-
if (audioOnly && settled && lag > AUDIO_STEER_LIMIT_SECONDS) {
34+
if (settled && lag <= MEDIA_SYNC_TOLERANCE_SECONDS) catchingUp.delete(el);
35+
const seekable = !catchingUp.has(el) || lag > MEDIA_HARD_SYNC_SECONDS;
36+
if (audioOnly && settled && lag > AUDIO_STEER_LIMIT_SECONDS && seekable) {
2937
steering.delete(el);
38+
catchingUp.add(el);
3039
return { rate: baseRate, seek: true };
3140
}
3241
const direction = Math.sign(offset);
@@ -44,6 +53,7 @@ export function steerMedia(
4453
/** Drop `el`'s steering: it was seeked, paused or replaced, so its old offset says nothing. */
4554
export function forgetSteering(el: HTMLMediaElement): void {
4655
steering.delete(el);
56+
catchingUp.delete(el);
4757
}
4858

4959
export function isSteering(el: HTMLMediaElement): boolean {

‎packages/player/tests/playbench/webkit-stops.ts‎

Lines changed: 40 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#!/usr/bin/env bun
2-
// Plays an edit in WebKit and counts how often the playhead stands still at the cuts.
3-
// Flags: --edit tentake|primary, --runs 3, --browser webkit|manual, --port 8099.
2+
// Plays an edit in WebKit and counts how often the playhead stands still at the cuts, how often audio is seeked,
3+
// and how much of the time sounding audio stands still.
4+
// Flags: --edit tentake|primary, --runs 3, --browser webkit|chromium|manual, --port 8099.
45
// webkit drives Playwright's WebKit (`playwright-core` and `playwright-core install webkit`); manual only serves
56
// /host.html and waits for a real Safari to open it, click Play and post the result.
67
import { existsSync } from "node:fs";
@@ -32,15 +33,17 @@ const HOST = `<!doctype html><meta charset="utf-8">
3233
<button id="go">Play</button>
3334
<hyperframes-player id="p" src="/comp.html" width="720" height="1280"></hyperframes-player>
3435
<script>
35-
var p = document.getElementById("p"), samples = [], t0 = 0, done = false;
36+
var p = document.getElementById("p"), samples = [], t0 = 0, done = false, seeks = 0;
3637
function finish() {
3738
if (done) return; done = true;
38-
var body = JSON.stringify({ t0: t0, end: performance.now(), samples: samples });
39+
var body = JSON.stringify({ t0: t0, end: performance.now(), samples: samples, seeks: seeks });
3940
fetch("/result", { method: "POST", body: body }).then(function () { window.__done = true; });
4041
}
4142
p.addEventListener("ended", finish);
4243
document.getElementById("go").onclick = function () {
4344
t0 = performance.now(); p.play();
45+
var doc = p.shadowRoot.querySelector("iframe").contentWindow.document;
46+
[].forEach.call(doc.querySelectorAll("audio"), function (a) { a.addEventListener("seeking", function () { seeks++; }); });
4447
(function tick() {
4548
if (done) return;
4649
var w = p.shadowRoot.querySelector("iframe").contentWindow, t = w.__player && w.__player.getTime ? w.__player.getTime() : p.currentTime;
@@ -53,7 +56,28 @@ const HOST = `<!doctype html><meta charset="utf-8">
5356
</script>
5457
<script src="/player.js"></script>`;
5558

56-
type Result = { t0: number; end: number; samples: [number, number, string?][] };
59+
type Result = { t0: number; end: number; samples: [number, number, string?][]; seeks?: number };
60+
61+
/** Share of sounding-audio time an element's clock stood still for more than STOP_MS. */
62+
function audioFrozenPct(s: Result["samples"]): number {
63+
let playing = 0;
64+
let frozen = 0;
65+
const since = new Map<string, { time: string; at: number; prev: number }>();
66+
for (const [at, , els] of s) {
67+
for (const entry of (els ?? "").split(" ").filter(Boolean)) {
68+
const [id, rest] = entry.split(":");
69+
const time = rest!.replace(/S?\d$/, "");
70+
const last = since.get(id!);
71+
if (last) playing += at - last.prev;
72+
if (!last || last.time !== time) since.set(id!, { time, at, prev: at });
73+
else {
74+
if (at - last.at > STOP_MS) frozen += at - Math.max(last.prev, last.at + STOP_MS);
75+
last.prev = at;
76+
}
77+
}
78+
}
79+
return playing > 0 ? Math.round((frozen / playing) * 1000) / 10 : 0;
80+
}
5781

5882
/** Stops are runs where the playhead stood still for more than STOP_MS, each tied to the cut it started at. */
5983
export function measure(r: Result, cuts: number[]) {
@@ -69,6 +93,8 @@ export function measure(r: Result, cuts: number[]) {
6993
const perCut = cuts.map((c) => stops.filter((x) => x.at >= c - 0.3 && x.at < c + 1.5));
7094
const played = (r.end - r.t0) / 1000;
7195
return {
96+
seeks: r.seeks ?? 0,
97+
audioFrozenPct: audioFrozenPct(s),
7298
stops: stops.length,
7399
stopsPerCut: stops.length / cuts.length,
74100
longestMs: Math.round(Math.max(0, ...stops.map((x) => x.ms))),
@@ -119,11 +145,14 @@ const server = Bun.serve({
119145
const cuts = edit.takes.slice(1).map((t) => t.start);
120146
const origin = `http://127.0.0.1:${server.port}`;
121147

122-
async function drivePlaywright() {
148+
async function drivePlaywright(engine: "webkit" | "chromium") {
123149
const name = "playwright-core"; // not a dependency: installed by hand, so kept out of the type check
124-
const { webkit } = (await import(name)) as { webkit: any }; // oxlint-disable-line no-explicit-any
125-
const browser = await webkit.launch();
126-
console.log(`WebKit ${browser.version()}`);
150+
const pw = (await import(name)) as Record<string, any>; // oxlint-disable-line no-explicit-any
151+
const browser = await pw[engine]!.launch({
152+
executablePath: process.env.PLAYBENCH_BROWSER_PATH,
153+
args: engine === "chromium" ? ["--autoplay-policy=no-user-gesture-required"] : [],
154+
});
155+
console.log(`${engine} ${browser.version()}`);
127156
for (let i = 0; i < RUNS; i++) {
128157
const page = await browser.newPage();
129158
await page.goto(`${origin}/host.html`);
@@ -137,7 +166,8 @@ async function drivePlaywright() {
137166
await browser.close();
138167
}
139168

140-
if (arg("browser", "webkit") === "webkit") await drivePlaywright();
169+
const engine = arg("browser", "webkit");
170+
if (engine === "webkit" || engine === "chromium") await drivePlaywright(engine);
141171
else {
142172
console.log(`open ${origin}/host.html in Safari, click Play (${RUNS} runs)`);
143173
while (results.length < RUNS) await new Promise<void>((r) => (resolveResult = r));

0 commit comments

Comments
 (0)