Skip to content

Commit 7d6c02f

Browse files
test(player): playbench measures first play, audio joins, automation, cuts and the autoplay fallback (heygen-com#6418)
* test(player): playbench measures first play, audio joins, automation, cuts and the autoplay fallback * fix(player): playbench takes-heard windows follow the composition playhead The host page has no player, so its samples carry no playhead. The windows fell back to nominal cut times.
1 parent 2be8bc2 commit 7d6c02f

9 files changed

Lines changed: 1238 additions & 2 deletions

File tree

‎packages/player/package.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,13 @@
2929
},
3030
"scripts": {
3131
"build": "tsup && node scripts/verify-runtime-pin.mjs",
32-
"typecheck": "tsc --noEmit && tsc --noEmit -p tests/perf/tsconfig.json",
32+
"typecheck": "tsc --noEmit && tsc --noEmit -p tests/perf/tsconfig.json && tsc --noEmit -p tests/playbench/tsconfig.json",
3333
"test": "vitest run",
3434
"test:browser-security": "bun run tests/browser/sandbox-origin.ts",
3535
"test:browser-raster": "bun run tests/browser/preview-raster-size.ts",
3636
"test:browser-box": "bun run tests/browser/player-box.ts",
37-
"perf": "bun run tests/perf/index.ts"
37+
"perf": "bun run tests/perf/index.ts",
38+
"playbench": "bun run tests/playbench/run.ts"
3839
},
3940
"dependencies": {
4041
"@hyperframes/core": "workspace:*"
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
_media/
2+
results/
Lines changed: 279 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,279 @@
1+
/** Turns one run's traces into numbers for the five playback problems; pure, so each is unit-tested. */
2+
import type { EditSpec } from "./compositions.ts";
3+
4+
export type Dump = {
5+
kind: "host" | "comp";
6+
events: [number, string, string, number | string][];
7+
frames: [number, string, number, number, number][];
8+
samples: [number, string, number, number, number, number, number, number | null][];
9+
params: [number, number | null, number | null, string, string, number, number | null][];
10+
edges: [number, number | string][];
11+
sources: [number, string][];
12+
tap: [number, number, number, number, number, number, number][];
13+
ctxs: [number, number, number, number][];
14+
rejects: number;
15+
};
16+
17+
export type Run = {
18+
edit: EditSpec;
19+
comp: Dump;
20+
host: Dump;
21+
playAt: number;
22+
endedAt: number | null;
23+
ownership: [number, string][];
24+
};
25+
26+
export type Metrics = {
27+
/** Problem 1 */
28+
firstTakeOverrunFrames: number;
29+
cutsWithOverrun: number;
30+
pictureAheadMs: number | null;
31+
playheadStartMs: number | null;
32+
/** Problem 2 */
33+
joins: number;
34+
joinsWithMidPlaySeek: number;
35+
jumpMedianMs: number | null;
36+
jumpMaxMs: number | null;
37+
audioLateStartMedianMs: number | null;
38+
joinsPreSeeked: number;
39+
/** Problem 3: scheduled lane time minus when the audio reached that point, in ms (negative is early). */
40+
automationOffsetMedianMs: number | null;
41+
automationOffsetMinMs: number | null;
42+
automationOffsetMaxMs: number | null;
43+
/** The end of each take's fade-out, the point a user hears land, one per take. */
44+
fadeOutOffsetsMs: number[];
45+
takesWithAutomationOff50: number;
46+
/** Problem 4 */
47+
secondFrameGapMedianMs: number | null;
48+
secondFrameGapMaxMs: number | null;
49+
playheadHoldMedianMs: number | null;
50+
playheadHoldP90Ms: number | null;
51+
holdsOver100: number;
52+
cuts: number;
53+
playedLongerMs: number | null;
54+
/** Problem 5: takes whose own stretch of the file was heard, how many parent-proxy seeks per second */
55+
takesHeard: number | null;
56+
proxySeeksPerSecond: number | null;
57+
/** How long the parent proxy played, from its first play to its last pause. */
58+
proxyPlayedSeconds: number | null;
59+
ownershipParent: boolean;
60+
rejectedPlays: number;
61+
};
62+
63+
export const median = (v: number[]): number | null => {
64+
if (!v.length) return null;
65+
const s = [...v].sort((a, b) => a - b);
66+
const m = s.length >> 1;
67+
return s.length % 2 ? s[m]! : (s[m - 1]! + s[m]!) / 2;
68+
};
69+
export const pct = (v: number[], p: number): number | null => {
70+
if (!v.length) return null;
71+
const s = [...v].sort((a, b) => a - b);
72+
return s[Math.min(s.length - 1, Math.floor(p * s.length))]!;
73+
};
74+
const round = (n: number | null, d = 1) => (n === null ? null : Math.round(n * 10 ** d) / 10 ** d);
75+
76+
type Sample = Dump["samples"][number];
77+
78+
/** First time the playhead reaches `ph` (epoch ms), from the per-rAF samples (one frame's rows share a time). */
79+
function timeAtPlayhead(samples: Sample[], ph: number): number | null {
80+
let prev: Sample | null = null;
81+
for (const b of samples) {
82+
if (b[7] === null) continue;
83+
if (prev && b[0] > prev[0] && prev[7]! <= ph && b[7] >= ph && b[7] > prev[7]!)
84+
return prev[0] + ((ph - prev[7]!) / (b[7] - prev[7]!)) * (b[0] - prev[0]);
85+
if (!prev || b[0] > prev[0]) prev = b;
86+
}
87+
return null;
88+
}
89+
90+
const byId = (samples: Sample[], id: string) => samples.filter((s) => s[1] === id);
91+
92+
/** Element-time to epoch: when did `currentTime` first reach `m` after the element was last seeked to before it. */
93+
function timeAtMedia(series: Sample[], m: number, after: number): number | null {
94+
for (let i = 1; i < series.length; i++) {
95+
const a = series[i - 1]!,
96+
b = series[i]!;
97+
if (b[0] < after || b[3] === 1) continue;
98+
if (a[2] <= m && b[2] >= m && b[2] > a[2])
99+
return a[0] + ((m - a[2]) / (b[2] - a[2])) * (b[0] - a[0]);
100+
}
101+
return null;
102+
}
103+
104+
export function analyze(run: Run): Metrics {
105+
const { edit, comp, host } = run;
106+
const fps = edit.fps;
107+
const takes = edit.takes;
108+
const samples = comp.samples;
109+
const m = {} as Metrics;
110+
111+
// ---- Problem 1: frames shown past an out-point, the picture running ahead of a clock that has not started
112+
const overrun = takes.map((t, k) => {
113+
const frames = comp.frames.filter((f) => f[1] === `v${k}` && f[4] === 1);
114+
const seen = new Set<number>();
115+
for (const f of frames)
116+
if (Math.round(f[2] * fps) >= t.outFrame) seen.add(Math.round(f[2] * fps));
117+
return seen.size;
118+
});
119+
m.firstTakeOverrunFrames = overrun[0] ?? 0;
120+
m.cutsWithOverrun = overrun.slice(1, -1).filter((n) => n > 0).length;
121+
const v0 = byId(samples, "v0");
122+
let ahead: number | null = null;
123+
for (const s of v0) {
124+
if (s[0] < run.playAt || s[0] > run.playAt + 1500 || s[3] === 1 || s[7] === null) continue;
125+
const d = (s[2] - takes[0]!.inPoint - s[7]) * 1000;
126+
ahead = ahead === null ? d : Math.max(ahead, d);
127+
}
128+
m.pictureAheadMs = round(ahead);
129+
const moved = samples.find((s) => s[0] >= run.playAt && s[7] !== null && s[7] > 0.05);
130+
m.playheadStartMs = moved ? round(moved[0] - run.playAt) : null;
131+
132+
// ---- Problem 2: audio joins
133+
const jumps: number[] = [];
134+
const lates: number[] = [];
135+
let withSeek = 0,
136+
preSeeked = 0,
137+
joins = 0;
138+
for (const t of takes.slice(1)) {
139+
const id = `a${t.index}`;
140+
const series = byId(samples, id);
141+
const t0 = timeAtPlayhead(samples, t.audio.start);
142+
if (t0 === null || !series.length) continue;
143+
joins++;
144+
const pre = series.find((s) => s[0] >= t0 - 50);
145+
if (pre && Math.abs(pre[2] - t.audio.mediaStart) < 0.02) preSeeked++;
146+
const playing = comp.events.find((e) => e[1] === id && e[2] === "playing" && e[0] >= t0 - 300);
147+
const from = playing ? Math.max(playing[0], t0) : t0;
148+
const seeks = comp.events.filter(
149+
(e) => e[1] === id && e[2] === "seeking" && e[0] >= from && e[0] <= t0 + 600,
150+
);
151+
if (seeks.length) withSeek++;
152+
for (const e of seeks) {
153+
const i = series.findIndex((s) => s[0] >= e[0]);
154+
if (i < 1) continue;
155+
const a = series[i - 1]!,
156+
b = series[i]!;
157+
const jump = (b[2] - a[2] - ((b[0] - a[0]) / 1000) * (b[5] || 1)) * 1000;
158+
jumps.push(Math.abs(jump));
159+
}
160+
const at = series.find((s) => s[0] >= t0 + 50 && s[3] === 0);
161+
if (at && at[7] !== null)
162+
lates.push((at[7] - t.audio.start + t.audio.mediaStart - at[2]) * 1000);
163+
}
164+
m.joins = joins;
165+
m.joinsWithMidPlaySeek = withSeek;
166+
m.jumpMedianMs = round(median(jumps));
167+
m.jumpMaxMs = round(jumps.length ? Math.max(...jumps) : null);
168+
m.audioLateStartMedianMs = round(median(lates));
169+
m.joinsPreSeeked = preSeeked;
170+
171+
// ---- Problem 3: where the volume lane landed against the audio it shapes
172+
// The gain node for an element is the first one downstream of its source that had params scheduled.
173+
const gainOfSource = new Map<string, number>();
174+
const owners = new Set(comp.params.map((p) => p[2]));
175+
for (const [src, el] of comp.sources) {
176+
const queue = [src];
177+
const visited = new Set<number>();
178+
while (queue.length) {
179+
const n = queue.shift()!;
180+
if (visited.has(n)) continue;
181+
visited.add(n);
182+
if (n !== src && owners.has(n)) {
183+
gainOfSource.set(el, n);
184+
break;
185+
}
186+
for (const [from, to] of comp.edges) if (from === n && typeof to === "number") queue.push(to);
187+
}
188+
}
189+
const offsets: number[] = [];
190+
const fadeOut: number[] = [];
191+
let off50 = 0;
192+
for (const t of takes) {
193+
const id = `a${t.index}`;
194+
const gain = gainOfSource.get(id);
195+
if (gain === undefined) continue;
196+
const calls = comp.params.filter((p) => p[2] === gain && p[4] === "linearRampToValueAtTime");
197+
const series = byId(samples, id);
198+
let bad = false;
199+
// The runtime sets the first point with setValueAtTime and ramps to each later one, in lane order.
200+
for (const [i, pt] of t.lane.entries()) {
201+
const target = calls[i - 1];
202+
if (!target || Math.abs((target[6] ?? NaN) - pt.v) > 0.0006) continue;
203+
const mPos = t.audio.mediaStart + pt.t;
204+
const e = timeAtMedia(series, mPos, run.playAt);
205+
if (e === null) continue;
206+
const ctxRow = [...comp.ctxs].reverse().find((c) => c[0] <= e) ?? comp.ctxs[0];
207+
if (!ctxRow) continue;
208+
const ctxAt = ctxRow[2] + (e - ctxRow[3]) / 1000;
209+
const off = (target[5] - ctxAt) * 1000;
210+
offsets.push(off);
211+
if (i === t.lane.length - 1) fadeOut.push(round(off) as number);
212+
if (Math.abs(off) > 50) bad = true;
213+
}
214+
if (bad) off50++;
215+
}
216+
m.automationOffsetMedianMs = round(median(offsets));
217+
m.automationOffsetMinMs = round(offsets.length ? Math.min(...offsets) : null);
218+
m.automationOffsetMaxMs = round(offsets.length ? Math.max(...offsets) : null);
219+
m.takesWithAutomationOff50 = off50;
220+
m.fadeOutOffsetsMs = fadeOut;
221+
222+
// ---- Problem 4: what a cut costs
223+
const gaps: number[] = [];
224+
const holds: number[] = [];
225+
for (const t of takes.slice(1)) {
226+
const fr = comp.frames
227+
.filter((f) => f[1] === `v${t.index}` && f[4] === 1)
228+
.sort((a, b) => a[0] - b[0]);
229+
if (fr.length >= 2) gaps.push(fr[1]![0] - fr[0]![0]);
230+
const a = timeAtPlayhead(samples, t.start - 0.5),
231+
b = timeAtPlayhead(samples, t.start + 0.5);
232+
if (a !== null && b !== null) holds.push(b - a - 1000);
233+
}
234+
m.cuts = takes.length - 1;
235+
m.secondFrameGapMedianMs = round(median(gaps));
236+
m.secondFrameGapMaxMs = round(gaps.length ? Math.max(...gaps) : null);
237+
m.playheadHoldMedianMs = round(median(holds));
238+
m.playheadHoldP90Ms = round(pct(holds, 0.9));
239+
m.holdsOver100 = holds.filter((h) => h > 100).length;
240+
m.playedLongerMs =
241+
run.endedAt === null ? null : round(run.endedAt - run.playAt - edit.duration * 1000);
242+
243+
// ---- Problem 5: the parent proxy path. Which stretch of the file reached the output, window by window.
244+
m.ownershipParent = run.ownership.some((o) => o[1] === "parent");
245+
m.rejectedPlays = comp.rejects;
246+
// Seeks per second of the time the parent proxy was actually playing (first play to its last pause).
247+
const firstPlay = host.events.find((e) => e[2] === "play" && e[0] >= run.playAt);
248+
const lastPause = [...host.events].reverse().find((e) => e[2] === "pause" || e[2] === "ended");
249+
const span =
250+
firstPlay && lastPause && lastPause[0] > firstPlay[0]
251+
? (lastPause[0] - firstPlay[0]) / 1000
252+
: null;
253+
const proxySeeks = host.events.filter(
254+
(e) => e[2] === "seeking" && firstPlay && e[0] >= firstPlay[0],
255+
).length;
256+
m.proxySeeksPerSecond = span ? round(proxySeeks / span) : null;
257+
m.proxyPlayedSeconds = round(span);
258+
const heardRows = [...host.tap, ...comp.tap].filter(
259+
(r) => r[4] >= 0 && r[5] >= 0 && r[6] >= run.playAt,
260+
);
261+
if (!heardRows.length) m.takesHeard = null;
262+
else {
263+
let heard = 0;
264+
for (const t of takes) {
265+
const startEpoch = timeAtPlayhead(samples, t.start) ?? run.playAt + t.start * 1000;
266+
const inWindow = heardRows.filter(
267+
(r) => r[6] >= startEpoch + 200 && r[6] <= startEpoch + t.dur * 1000,
268+
);
269+
if (!inWindow.length) continue;
270+
const right = inWindow.filter((r) => {
271+
const pos = r[5] * 16 + r[4] * 0.5;
272+
return pos >= t.inPoint - 0.75 && pos <= t.inPoint + t.dur + 0.75;
273+
});
274+
if (right.length / inWindow.length > 0.5) heard++;
275+
}
276+
m.takesHeard = heard;
277+
}
278+
return m;
279+
}

0 commit comments

Comments
 (0)