Skip to content

Commit 7f452ad

Browse files
feat(core): speed ramps as a rate lane in data-automation (#4129)
* feat(core): speed ramps as a rate lane in data-automation A clip can carry a rate lane in data-automation. sourceTimeAt integrates it to source time and is the one mapping the runtime uses for media seeking, playback rate, natural duration, child timelines and the audio-master clock. Rate is clamped to 0.1-10 in one place, pitch is preserved unless data-preserve-pitch is false, and ramped clips play through the media element. Lint warns on a rate lane outside video and audio. * fix(core): format the speed-ramp files and expect the unified rate clamp in the engine tests * test(engine): read the playback-rate clamp from core instead of a literal
1 parent 8666720 commit 7f452ad

20 files changed

Lines changed: 537 additions & 34 deletions

‎packages/core/src/audioAutomation.ts‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
import { getAudioFxDef, type HfAudioFxChain } from "./audioFx.js";
1515
import { MAX_AUDIO_GAIN } from "./audioGain.js";
16+
import { MAX_PLAYBACK_RATE, MIN_PLAYBACK_RATE } from "./playbackRateBounds.js";
1617

1718
export const HF_AUDIO_AUTOMATION_ATTR = "data-automation";
1819

@@ -80,14 +81,19 @@ export class AudioAutomationError extends Error {
8081

8182
export const VOLUME_TARGET = "volume";
8283

84+
/** Playback-rate lane: `v` is a speed multiplier over clip-local time. See `speedRamp.ts`. */
85+
export const RATE_TARGET = "rate";
86+
8387
export type HfAutomationTarget =
8488
| { kind: "volume" }
89+
| { kind: "rate" }
8590
| { kind: "fx"; nodeId: string; param: string }
8691
| { kind: "preset"; presetId: string };
8792

8893
/** Split a target string. Returns null for anything unrecognised. */
8994
export function parseAutomationTarget(target: string): HfAutomationTarget | null {
9095
if (target === VOLUME_TARGET) return { kind: "volume" };
96+
if (target === RATE_TARGET) return { kind: "rate" };
9197
const parts = target.split(".");
9298
// `fx.preset.<id>` before the 3-part fx form, because it IS a 3-part fx form
9399
// with a reserved node id — an effect can never be called "preset", since ids
@@ -169,6 +175,16 @@ export const VOLUME_RANGE: AutomationRange = {
169175
default: 1,
170176
};
171177

178+
export const RATE_RANGE: AutomationRange = {
179+
min: MIN_PLAYBACK_RATE,
180+
max: MAX_PLAYBACK_RATE,
181+
step: 0.05,
182+
unit: "x",
183+
label: "Speed",
184+
scale: "log",
185+
default: 1,
186+
};
187+
172188
/**
173189
* Resolve a lane's target against a chain. Returns null when the target names
174190
* a node or parameter that is not there — the effect was deleted, or the
@@ -181,6 +197,7 @@ export function resolveAutomationRange(
181197
const parsed = parseAutomationTarget(target);
182198
if (!parsed) return null;
183199
if (parsed.kind === "volume") return VOLUME_RANGE;
200+
if (parsed.kind === "rate") return RATE_RANGE;
184201
if (parsed.kind === "preset") {
185202
// Only for a preset the chain actually carries, so a lane left behind by a
186203
// removed preset resolves to nothing and is dropped at read time — the same
@@ -335,7 +352,12 @@ export function normalizeAutomation(automation: HfAutomation): HfAutomation {
335352
const lanes: HfAutomationLane[] = [];
336353
for (const lane of automation.lanes) {
337354
if (!parseAutomationTarget(lane.target)) continue;
338-
const range = lane.target === VOLUME_TARGET ? VOLUME_RANGE : null;
355+
const range =
356+
lane.target === VOLUME_TARGET
357+
? VOLUME_RANGE
358+
: lane.target === RATE_TARGET
359+
? RATE_RANGE
360+
: null;
339361
const points = normalizePoints(lane.points ?? [], range);
340362
if (points.length > 0) lanes.push({ target: lane.target, points });
341363
}
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
/** The one clamp for a playback rate: the constant attribute and the rate lane alike. */
2+
export const MIN_PLAYBACK_RATE = 0.1;
3+
export const MAX_PLAYBACK_RATE = 10;

‎packages/core/src/runtime/audioAutomationVolume.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,18 +24,20 @@ import {
2424
const cache = new Map<string, HfAutomationLane | null>();
2525
const CACHE_LIMIT = 64;
2626

27-
function laneFromAttr(raw: string): HfAutomationLane | null {
28-
const hit = cache.get(raw);
27+
/** The lane for `target` inside a `data-automation` value, or null when absent or unreadable. */
28+
export function laneFromAttr(raw: string, target: string = VOLUME_TARGET): HfAutomationLane | null {
29+
const key = `${target}\0${raw}`;
30+
const hit = cache.get(key);
2931
if (hit !== undefined) return hit;
3032
let lane: HfAutomationLane | null = null;
3133
try {
32-
lane = parseAutomation(raw).lanes.find((l) => l.target === VOLUME_TARGET) ?? null;
34+
lane = parseAutomation(raw).lanes.find((l) => l.target === target) ?? null;
3335
} catch {
3436
// Unreadable automation plays the track flat rather than silencing it.
3537
lane = null;
3638
}
3739
if (cache.size > CACHE_LIMIT) cache.clear();
38-
cache.set(raw, lane);
40+
cache.set(key, lane);
3941
return lane;
4042
}
4143

‎packages/core/src/runtime/clock.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,21 @@ describe("TransportClock", () => {
272272
expect(clock.getSource()).toBe("monotonic");
273273
});
274274

275+
it("maps audio position back to composition time through a rate lane", () => {
276+
const { clock } = createClock({ duration: 20 });
277+
const rate = {
278+
target: "rate",
279+
points: [
280+
{ t: 0, v: 1 },
281+
{ t: 2, v: 3 },
282+
],
283+
};
284+
const audioEl = createMockAudioEl(4, false);
285+
clock.play();
286+
clock.attachAudioSource({ el: audioEl, compositionStart: 1, mediaStart: 0, rate });
287+
expect(clock.now()).toBeCloseTo(1 + 2 + (4 - 3.641) / 3, 2);
288+
});
289+
275290
it("accounts for compositionStart offset", () => {
276291
const { clock } = createClock({ duration: 20 });
277292
const audioEl = createMockAudioEl(2.0, false);

‎packages/core/src/runtime/clock.ts‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import { timeAtSourceTime, type RateSpec } from "../speedRamp.js";
2+
13
export type TransportClockSnapshot = {
24
time: number;
35
playing: boolean;
@@ -11,6 +13,8 @@ export type AudioClockSource =
1113
el: HTMLMediaElement;
1214
compositionStart: number;
1315
mediaStart: number;
16+
/** The clip's rate lane; a constant rate is read from `el.playbackRate`. */
17+
rate?: RateSpec;
1418
}
1519
| {
1620
currentTimeSeconds: number;
@@ -54,12 +58,14 @@ export class TransportClock {
5458
if ("currentTimeSeconds" in this._audioSource) {
5559
audioTime = this._audioSource.currentTimeSeconds;
5660
} else {
57-
const { el, compositionStart, mediaStart } = this._audioSource;
61+
const { el, compositionStart, mediaStart, rate } = this._audioSource;
5862
if (!el.paused && Number.isFinite(el.currentTime)) {
5963
audioTime =
60-
((el.currentTime - mediaStart) / (el.playbackRate > 0 ? el.playbackRate : 1)) *
61-
this._rate +
62-
compositionStart;
64+
typeof rate === "object"
65+
? timeAtSourceTime(rate, el.currentTime - mediaStart) + compositionStart
66+
: ((el.currentTime - mediaStart) / (el.playbackRate > 0 ? el.playbackRate : 1)) *
67+
this._rate +
68+
compositionStart;
6369
}
6470
}
6571
if (audioTime !== null) {

‎packages/core/src/runtime/init.test.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4171,6 +4171,20 @@ describe("derived duration floor recomputation", () => {
41714171
expect(await runFrames(1)).toBe(25);
41724172
});
41734173

4174+
it("re-derives when a speed-ramp lane is edited", async () => {
4175+
mountComposition(`<video data-start="0"></video>`);
4176+
const video = document.querySelector("video")!;
4177+
setNativeDuration(video, 10);
4178+
initSandboxRuntimeModular();
4179+
expect(await runFrames(2)).toBe(10);
4180+
4181+
video.setAttribute(
4182+
"data-automation",
4183+
JSON.stringify({ version: 1, lanes: [{ target: "rate", points: [{ t: 0, v: 2 }] }] }),
4184+
);
4185+
expect(await runFrames(1)).toBe(5);
4186+
});
4187+
41744188
it("re-derives when a clip is moved later on the timeline", async () => {
41754189
mountComposition(`<video data-start="0" data-duration="10"></video>`);
41764190
initSandboxRuntimeModular();

‎packages/core/src/runtime/init.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,11 @@ import {
1818
patchWebGLVideoTextureCompat,
1919
} from "./adapters/video-texture-compat";
2020
import { forceDispatchSeekEvent, waitForSeekCompletion } from "./adapters/seek-dispatch";
21+
import { sourceTimeAt } from "../speedRamp";
2122
import { createWaapiAdapter } from "./adapters/waapi";
2223
import {
2324
readElementPlaybackRate,
25+
readElementRateSpec,
2426
readElementPlaybackStart,
2527
refreshRuntimeMediaCache,
2628
resolveRuntimeMediaClipDuration,
@@ -1068,6 +1070,7 @@ export function initSandboxRuntimeModular(): void {
10681070
"data-hf-auto-start",
10691071
MEDIA_START_BASIS_ATTR,
10701072
"data-playback-rate",
1073+
"data-automation",
10711074
"data-playback-start",
10721075
"data-media-start",
10731076
// `data-start` may be an expression referencing another element by id, and
@@ -3366,7 +3369,7 @@ export function initSandboxRuntimeModular(): void {
33663369
const timelineDuration = getTimelineDurationSeconds(timeline);
33673370
const sourceTime =
33683371
readElementPlaybackStart(node) +
3369-
Math.max(0, timeSeconds - start) * readElementPlaybackRate(node);
3372+
sourceTimeAt(readElementRateSpec(node), Math.max(0, timeSeconds - start));
33703373
const localTime = Math.max(
33713374
0,
33723375
timelineDuration != null && timelineDuration > 0
@@ -3800,7 +3803,12 @@ export function initSandboxRuntimeModular(): void {
38003803
const mediaStart = readElementPlaybackStart(rawEl);
38013804
if (Number.isFinite(start) && state.currentTime >= start && state.currentTime < end) {
38023805
if (!rawEl.paused) {
3803-
clock.attachAudioSource({ el: rawEl, compositionStart: start, mediaStart });
3806+
clock.attachAudioSource({
3807+
el: rawEl,
3808+
compositionStart: start,
3809+
mediaStart,
3810+
rate: readElementRateSpec(rawEl),
3811+
});
38043812
foundActive = true;
38053813
} else if (!rawEl.error && rawEl.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) {
38063814
// Audio is buffering — freeze visuals at last known position
@@ -3953,6 +3961,8 @@ export function initSandboxRuntimeModular(): void {
39533961
// that existed before (#3458).
39543962
const route = classifyWebAudioMediaRoute(rawEl);
39553963
reportWebAudioMediaRoute(rawEl, route);
3964+
// Decoded buffers cannot follow a rate curve without shifting pitch; the media element can.
3965+
if (typeof readElementRateSpec(rawEl) !== "number") continue;
39563966
// The cross-origin verdict's BEST outcome is decode, since a CDN that
39573967
// sends `Access-Control-Allow-Origin` (the author just never wrote the
39583968
// `crossorigin` attribute) decodes fine and keeps the whole FX graph.

‎packages/core/src/runtime/media.test.ts‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,12 @@ describe("readElementPlaybackRate", () => {
4242
expect(readElementPlaybackRate(el)).toBe(1);
4343
});
4444

45-
it("clamps to [0.1, 5]", () => {
45+
it("clamps to [0.1, 10]", () => {
4646
const el = document.createElement("video");
4747
Object.defineProperty(el, "defaultPlaybackRate", { value: 0.01, writable: true });
4848
expect(readElementPlaybackRate(el)).toBe(0.1);
49-
Object.defineProperty(el, "defaultPlaybackRate", { value: 10, writable: true });
50-
expect(readElementPlaybackRate(el)).toBe(5);
49+
Object.defineProperty(el, "defaultPlaybackRate", { value: 20, writable: true });
50+
expect(readElementPlaybackRate(el)).toBe(10);
5151
});
5252

5353
it("defaults to 1 for NaN/negative/zero", () => {
@@ -165,16 +165,16 @@ describe("refreshRuntimeMediaCache", () => {
165165
expect(result.mediaClips[0].playbackRate).toBe(1);
166166
});
167167

168-
it("clamps playback rate to [0.1, 5]", () => {
168+
it("clamps playback rate to [0.1, 10]", () => {
169169
const el1 = createVideo({ "data-start": "0", "data-duration": "5" });
170170
Object.defineProperty(el1, "defaultPlaybackRate", { value: 0.01, writable: true });
171171
const r1 = refreshRuntimeMediaCache();
172172
expect(r1.mediaClips[0].playbackRate).toBe(0.1);
173173
document.body.innerHTML = "";
174174
const el2 = createVideo({ "data-start": "0", "data-duration": "5" });
175-
Object.defineProperty(el2, "defaultPlaybackRate", { value: 10, writable: true });
175+
Object.defineProperty(el2, "defaultPlaybackRate", { value: 20, writable: true });
176176
const r2 = refreshRuntimeMediaCache();
177-
expect(r2.mediaClips[0].playbackRate).toBe(5);
177+
expect(r2.mediaClips[0].playbackRate).toBe(10);
178178
});
179179

180180
it("adjusts fallback duration by playback rate", () => {
@@ -365,6 +365,25 @@ describe("syncRuntimeMedia", () => {
365365
document.body.innerHTML = "";
366366
});
367367

368+
describe("speed ramp", () => {
369+
it("seeks to the integrated source time and plays at the instantaneous rate", () => {
370+
const rate = {
371+
target: "rate",
372+
points: [
373+
{ t: 0, v: 1 },
374+
{ t: 2, v: 3 },
375+
],
376+
};
377+
const clip = createMockClip({ start: 1, end: 5, rate });
378+
Object.defineProperty(clip.el, "readyState", { value: 4, writable: true });
379+
syncRuntimeMedia({ clips: [clip], timeSeconds: 3, playing: false, playbackRate: 1 });
380+
expect(clip.el.currentTime).toBeCloseTo(3.641, 2);
381+
syncRuntimeMedia({ clips: [clip], timeSeconds: 3, playing: true, playbackRate: 1 });
382+
expect(clip.el.playbackRate).toBeCloseTo(3, 5);
383+
expect(clip.el.preservesPitch).toBe(true);
384+
});
385+
});
386+
368387
describe("volume automation lane", () => {
369388
const DUCK = JSON.stringify({
370389
version: 1,

‎packages/core/src/runtime/media.ts‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,22 @@
11
import { swallow } from "./diagnostics";
22
import { interpolateVolumeGain, type VolumeKeyframe } from "./mediaVolumeEnvelope.js";
33
import { elementVolumeLaneGain } from "./audioAutomationVolume.js";
4-
import { readElementPlaybackRate, readMediaStart } from "./playbackRate.js";
4+
import { readElementPlaybackRate, readElementRateSpec, readMediaStart } from "./playbackRate.js";
5+
import {
6+
rateAt,
7+
readPreservePitch,
8+
sourceTimeAt,
9+
timeAtSourceTime,
10+
type RateSpec,
11+
} from "../speedRamp.js";
512
import { clampAudioGain } from "../audioGain.js";
613
import { isMemberGroupHidden } from "../audioGroups.js";
714
import { findInjectedRenderFrame } from "./renderFrameSibling.js";
8-
export { readElementPlaybackRate, resolveNaturalMediaTimelineDuration } from "./playbackRate.js";
15+
export {
16+
readElementPlaybackRate,
17+
readElementRateSpec,
18+
resolveNaturalMediaTimelineDuration,
19+
} from "./playbackRate.js";
920

1021
export function readElementPlaybackStart(el: Element): number {
1122
return readMediaStart(el);
@@ -43,6 +54,8 @@ export type RuntimeMediaClip = {
4354
end: number;
4455
volume: number | null;
4556
playbackRate: number;
57+
/** The rate lane when the clip has one; otherwise `playbackRate`. */
58+
rate?: RateSpec;
4659
loop: boolean;
4760
/** Source media duration in seconds (from el.duration). Used for loop wrapping. */
4861
sourceDuration: number | null;
@@ -94,14 +107,15 @@ export function refreshRuntimeMediaCache(params?: {
94107
if (!Number.isFinite(start)) continue;
95108
const mediaStart = readElementPlaybackStart(el);
96109
const playbackRate = readElementPlaybackRate(el);
110+
const rate = readElementRateSpec(el);
97111
const loop = el.loop;
98112
const sourceDuration = Number.isFinite(el.duration) && el.duration > 0 ? el.duration : null;
99113
let duration =
100114
params?.resolveDurationSeconds?.(el) ?? Number.parseFloat(el.dataset.duration ?? "");
101115
if ((!Number.isFinite(duration) || duration < 0) && sourceDuration != null) {
102116
// Effective duration accounts for playback rate:
103117
// at 0.5x, a 10s source plays for 20s on the timeline
104-
duration = Math.max(0, (sourceDuration - mediaStart) / playbackRate);
118+
duration = Math.max(0, timeAtSourceTime(rate, sourceDuration - mediaStart));
105119
}
106120
const hasKnownDuration = Number.isFinite(duration) && duration >= 0;
107121
const end = hasKnownDuration ? start + duration : Number.POSITIVE_INFINITY;
@@ -114,6 +128,7 @@ export function refreshRuntimeMediaCache(params?: {
114128
end,
115129
volume: Number.isFinite(volumeRaw) ? volumeRaw : null,
116130
playbackRate,
131+
rate,
117132
loop,
118133
sourceDuration,
119134
};
@@ -243,7 +258,8 @@ export function syncRuntimeMedia(params: {
243258
for (const clip of params.clips) {
244259
const { el } = clip;
245260
if (!el.isConnected) continue;
246-
let relTime = (params.timeSeconds - clip.start) * clip.playbackRate + clip.mediaStart;
261+
const clipRate = clip.rate ?? clip.playbackRate;
262+
let relTime = sourceTimeAt(clipRate, params.timeSeconds - clip.start) + clip.mediaStart;
247263
const isNonLoopVideo = el.tagName === "VIDEO" && !clip.loop;
248264
const isHeldVideoTail =
249265
isNonLoopVideo &&
@@ -369,7 +385,8 @@ export function syncRuntimeMedia(params: {
369385
if (el.preload !== "auto") el.preload = "auto";
370386
try {
371387
// Per-element rate × global transport rate
372-
el.playbackRate = clip.playbackRate * params.playbackRate;
388+
el.playbackRate = rateAt(clipRate, params.timeSeconds - clip.start) * params.playbackRate;
389+
el.preservesPitch = readPreservePitch(el);
373390
} catch (err) {
374391
// ignore unsupported playbackRate
375392
swallow("runtime.media.site1", err);

‎packages/core/src/runtime/playbackRate.test.ts‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
import { describe, expect, it } from "vitest";
2-
import { resolveNaturalMediaTimelineDuration } from "./playbackRate";
2+
import {
3+
resolveNaturalMediaTimelineDuration,
4+
resolveNaturalMediaTimelineDurationFromValues,
5+
} from "./playbackRate";
36

47
function elementWith(attributes: Record<string, string>): Pick<Element, "getAttribute"> {
58
return {
@@ -29,3 +32,23 @@ describe("resolveNaturalMediaTimelineDuration", () => {
2932
expect(resolveNaturalMediaTimelineDuration(elementWith({}), Number.NaN)).toBeNull();
3033
});
3134
});
35+
36+
describe("rate lane duration", () => {
37+
it("resolves natural media duration through the lane", () => {
38+
const lane = {
39+
target: "rate",
40+
points: [
41+
{ t: 0, v: 1 },
42+
{ t: 2, v: 3 },
43+
],
44+
};
45+
expect(resolveNaturalMediaTimelineDurationFromValues(4, 0, lane)).toBeCloseTo(
46+
2 + (4 - 3.641) / 3,
47+
2,
48+
);
49+
expect(resolveNaturalMediaTimelineDurationFromValues(10, 0, lane)).toBeCloseTo(
50+
2 + (10 - 3.641) / 3,
51+
2,
52+
);
53+
});
54+
});

0 commit comments

Comments
 (0)