Skip to content

Commit 7fae35f

Browse files
fix(studio): carve and duck follow each clip's trim and speed (#4661)
* fix(studio): read and write a clip's in-point the way playback does * fix(studio): a group head trim keeps the minimum clip length off the grid * fix(studio): a group head trim stops where a single-clip trim does and never goes below zero * fix(studio): group members on the grid move together when a trim stops at its limit * fix(studio): level-out measures and places its lift on the clip clock at any speed * fix(studio): round level-out lane times and tighten the half-speed test * fix(studio): carve and duck follow each clip's trim and speed * fix(studio): carve ignores speech after the bed stops playing * test(studio): carve ignores speech past a bed that ducks and has no duration * fix(studio): carve cuts speech past the bed's end at every strength
1 parent addc72d commit 7fae35f

6 files changed

Lines changed: 319 additions & 73 deletions

File tree

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
import { describe, expect, it } from "vitest";
2+
import { clipAudioOnItsClock, readClipClock } from "./clipAudioClock";
3+
4+
const RATE = 48000;
5+
const sine = (seconds: number, hz: number) =>
6+
Float32Array.from({ length: RATE * seconds }, (_, i) => Math.sin((2 * Math.PI * hz * i) / RATE));
7+
const zeroCrossingsPerSecond = (x: Float32Array) => {
8+
let n = 0;
9+
for (let i = 1; i < x.length; i += 1) if (x[i - 1]! < 0 !== x[i]! < 0) n += 1;
10+
return n / (x.length / RATE);
11+
};
12+
13+
describe("readClipClock", () => {
14+
it("reads the in-point, the rate lane over the constant, and a positive duration", () => {
15+
const attrs: Record<string, string> = {
16+
"data-playback-start": "1.5",
17+
"data-media-start": "9",
18+
"data-playback-rate": "3",
19+
"data-automation":
20+
'{"version":1,"lanes":[{"target":"rate","points":[{"t":0,"v":2},{"t":4,"v":2}]}]}',
21+
"data-duration": "4",
22+
};
23+
const clock = readClipClock((name) => attrs[name]);
24+
expect({ ...clock, rate: typeof clock.rate }).toEqual({
25+
inPoint: 1.5,
26+
rate: "object",
27+
duration: 4,
28+
});
29+
expect(readClipClock((name) => (name === "data-duration" ? "0" : null))).toEqual({
30+
inPoint: 0,
31+
rate: 1,
32+
duration: null,
33+
});
34+
});
35+
});
36+
37+
describe("clipAudioOnItsClock", () => {
38+
it("is the exact slice the clip plays at rate 1", () => {
39+
const samples = sine(4, 440);
40+
const out = clipAudioOnItsClock(samples, RATE, { inPoint: 1, rate: 1, duration: 2 });
41+
expect(Array.from(out)).toEqual(Array.from(samples.subarray(RATE, 3 * RATE)));
42+
});
43+
44+
it("lasts the clip's duration at 2x, consuming twice as much source", () => {
45+
// Each sample holds its own source time, so the output reads back where it was taken from.
46+
const samples = Float32Array.from({ length: RATE * 6 }, (_, i) => i / RATE);
47+
const out = clipAudioOnItsClock(samples, RATE, { inPoint: 1, rate: 2, duration: 2 });
48+
expect({ length: out.length, atOneSecond: Number(out[RATE]!.toFixed(1)) }).toEqual({
49+
length: 2 * RATE,
50+
atOneSecond: 3,
51+
});
52+
});
53+
54+
it("keeps the pitch at 2x and 0.5x", () => {
55+
const samples = sine(4, 440);
56+
for (const rate of [2, 0.5]) {
57+
const out = clipAudioOnItsClock(samples, RATE, { inPoint: 0, rate, duration: 1.5 });
58+
expect(zeroCrossingsPerSecond(out) / 880).toBeCloseTo(1, 1);
59+
}
60+
});
61+
});
Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
import {
2+
resolveRateSpec,
3+
sourceTimeAt,
4+
timeAtSourceTime,
5+
type RateSpec,
6+
} from "@hyperframes/core/speed-ramp";
7+
import {
8+
readDataDurationSeconds,
9+
readMediaOffsetSeconds,
10+
readPlaybackRate,
11+
type AttrReader,
12+
} from "@hyperframes/parsers/media-duration";
13+
14+
/** Where a clip's audio starts in its file, how fast it plays, and for how long (null: to the end). */
15+
export interface ClipClock {
16+
inPoint: number;
17+
rate: RateSpec;
18+
duration: number | null;
19+
}
20+
21+
export function readClipClock(getAttr: AttrReader): ClipClock {
22+
return {
23+
inPoint: readMediaOffsetSeconds(getAttr),
24+
rate: resolveRateSpec(getAttr("data-automation"), readPlaybackRate(getAttr)),
25+
duration: readDataDurationSeconds(getAttr),
26+
};
27+
}
28+
29+
/** The source samples a clip plays, still on the file's clock. */
30+
export function clipSourceWindow(
31+
samples: Float32Array,
32+
sampleRate: number,
33+
clock: ClipClock,
34+
): Float32Array {
35+
const from = Math.min(samples.length, Math.floor(clock.inPoint * sampleRate));
36+
const to = clock.duration
37+
? Math.min(
38+
samples.length,
39+
from + Math.ceil(sourceTimeAt(clock.rate, clock.duration) * sampleRate),
40+
)
41+
: samples.length;
42+
return samples.subarray(from, to);
43+
}
44+
45+
const GRAIN = 4096;
46+
const HOP = GRAIN / 2;
47+
const HANN = Float32Array.from(
48+
{ length: GRAIN },
49+
(_, k) => 0.5 - 0.5 * Math.cos((2 * Math.PI * k) / GRAIN),
50+
);
51+
52+
/**
53+
* The audio heard over clip time [0, duration]. Off rate 1, Hann grains overlap-added at the source
54+
* time each lands on: pitch kept, like export's atempo, close enough for band and level analysis.
55+
*/
56+
export function clipAudioOnItsClock(
57+
samples: Float32Array,
58+
sampleRate: number,
59+
clock: ClipClock,
60+
): Float32Array {
61+
const source = clipSourceWindow(samples, sampleRate, clock);
62+
if (clock.rate === 1) return source;
63+
const out = new Float32Array(
64+
Math.round(timeAtSourceTime(clock.rate, source.length / sampleRate) * sampleRate),
65+
);
66+
// Grains are not phase-aligned, so off rate 1 the level reads ~1.3 dB low (up to ~3 dB on
67+
// tones); a WSOLA stretch would keep it.
68+
for (let at = -HOP; at < out.length; at += HOP) {
69+
const centre = Math.round(sourceTimeAt(clock.rate, (at + HOP) / sampleRate) * sampleRate);
70+
for (let k = 0; k < GRAIN; k += 1) {
71+
const o = at + k;
72+
const s = centre - HOP + k;
73+
if (o < 0 || o >= out.length || s < 0 || s >= source.length) continue;
74+
out[o] = (out[o] ?? 0) + (HANN[k] ?? 0) * (source[s] ?? 0);
75+
}
76+
}
77+
return out;
78+
}

‎packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -989,6 +989,141 @@ describe("AudioFxGroup dynamic carve", () => {
989989
expect(onSetAttributeLive.mock.calls.every((c) => c[0] === "data-fx-carve")).toBe(true);
990990
expect(onSetAttributeQuiet.mock.calls.some((c) => c[0] === "data-fx-chain")).toBe(false);
991991
});
992+
993+
describe("on each clip's own clock", () => {
994+
type Point = { t: number; v: number };
995+
/** `hz` at `amp` over [from, to) of a `seconds`-long file, silence elsewhere. */
996+
const tone = (seconds: number, [from, to]: [number, number], hz = 1000, amp = 0.7) =>
997+
Float32Array.from({ length: SAMPLE_RATE * seconds }, (_, i) => {
998+
const t = i / SAMPLE_RATE;
999+
return t >= from && t < to ? amp * Math.sin(2 * Math.PI * hz * t) : 0;
1000+
});
1001+
1002+
/** Each file decodes to its own samples, found by the URL that was fetched. */
1003+
function stubFiles(files: Record<string, Float32Array>): void {
1004+
vi.stubGlobal(
1005+
"fetch",
1006+
vi.fn(async (url: string) => ({ arrayBuffer: async () => url })),
1007+
);
1008+
vi.stubGlobal(
1009+
"OfflineAudioContext",
1010+
class {
1011+
decodeAudioData = async (url: string) => {
1012+
const data = Object.entries(files).find(([name]) => url.endsWith(name))![1];
1013+
return { sampleRate: SAMPLE_RATE, getChannelData: () => data };
1014+
};
1015+
},
1016+
);
1017+
}
1018+
1019+
const valueAt = (points: Point[], t: number): number => {
1020+
const i = points.findIndex((p) => p.t >= t);
1021+
if (i <= 0) return points.at(i)!.v;
1022+
const [a, b] = [points[i - 1]!, points[i]!];
1023+
return a.v + ((b.v - a.v) * (t - a.t)) / (b.t - a.t);
1024+
};
1025+
1026+
/** Carves the bed against `vo` and reads back the lane driving the first node of `type`. */
1027+
async function carveLane(
1028+
type: string,
1029+
voice: Record<string, string>,
1030+
bed: Record<string, string> = {},
1031+
strength = 0,
1032+
) {
1033+
const { host, onSetAttributeQuiet } = mount({
1034+
"fx-chain": carvedChain,
1035+
"fx-carve": settings(true, { strength, sources: [] }),
1036+
start: "0",
1037+
...bed,
1038+
});
1039+
const vo = document.getElementById("vo")!;
1040+
vo.setAttribute("src", "voice.wav");
1041+
for (const [name, value] of Object.entries(voice)) vo.setAttribute(`data-${name}`, value);
1042+
document.getElementById("bed")!.setAttribute("src", "bed.wav");
1043+
await act(async () => {
1044+
pickSource(host, "vo");
1045+
});
1046+
const calls = onSetAttributeQuiet.mock.calls;
1047+
const nodes = parseWrite(writeTo(calls, "data-fx-chain")!).nodes as {
1048+
id: string;
1049+
type: string;
1050+
params: { gain: number; frequency: number };
1051+
}[];
1052+
const node = nodes.find((n) => n.type === type)!;
1053+
const lanes = parseWrite(writeTo(calls, "data-automation")!).lanes as {
1054+
target: string;
1055+
points: Point[];
1056+
}[];
1057+
const points = lanes.find((l) => l.target === `fx.${node.id}.gain`)!.points;
1058+
return {
1059+
depth: node.params.gain,
1060+
frequency: node.params.frequency,
1061+
points,
1062+
at: (t: number) => valueAt(points, t),
1063+
};
1064+
}
1065+
1066+
/** Quiet 1 kHz speech at 0-2 s, loud 4 kHz speech at 4-6 s, in an 8 s file. */
1067+
const lateSpeech = () => {
1068+
const after = tone(8, [4, 6], 4000, 0.9);
1069+
return tone(8, [0, 2], 1000, 0.3).map((v, i) => v + after[i]!);
1070+
};
1071+
1072+
it("ignores speech after the bed has stopped playing", async () => {
1073+
// The bed file runs 8 s, but the clip plays only its first 3 s.
1074+
stubFiles({ "voice.wav": lateSpeech(), "bed.wav": tone(8, [0, 8], 200) });
1075+
const lane = await carveLane("peaking", { start: "0" }, { duration: "3" });
1076+
expect(lane.frequency).toBe(1000);
1077+
});
1078+
1079+
it("ignores speech after a bed with no duration runs out, when it ducks", async () => {
1080+
// No data-duration: the bed plays its 3 s file. Strength 0.05 ducks and keeps one band.
1081+
stubFiles({ "voice.wav": lateSpeech(), "bed.wav": tone(3, [0, 3], 200) });
1082+
const lane = await carveLane("peaking", { start: "0" }, {}, 0.05);
1083+
expect(lane.frequency).toBe(1000);
1084+
});
1085+
1086+
it.each([
1087+
["no duration", {}],
1088+
["a duration longer than its file", { duration: "6" }],
1089+
])("ignores speech after a 3 s bed file with %s, without ducking", async (_, bed) => {
1090+
stubFiles({ "voice.wav": lateSpeech(), "bed.wav": tone(3, [0, 3], 200) });
1091+
const lane = await carveLane("peaking", { start: "0" }, bed);
1092+
expect(lane.frequency).toBe(1000);
1093+
});
1094+
1095+
it("cuts where a trimmed voice speaks, not where its file does", async () => {
1096+
// File speech 2-3.5 s; the clip plays 2-4 s of it, so the bed hears it at 0-1.5 s.
1097+
stubFiles({ "voice.wav": tone(6, [2, 3.5]), "bed.wav": tone(8, [0, 8], 200) });
1098+
const lane = await carveLane("peaking", { start: "0", "media-start": "2", duration: "2" });
1099+
expect({ speaking: lane.at(0.8) <= lane.depth * 0.5, after: lane.at(3) }).toEqual({
1100+
speaking: true,
1101+
after: 0,
1102+
});
1103+
});
1104+
1105+
it("cuts over half the source length for a voice at 2x", async () => {
1106+
// File speech 1-3 s at 2x: the bed hears it at 0.5-1.5 s.
1107+
stubFiles({ "voice.wav": tone(4, [1, 3]), "bed.wav": tone(8, [0, 8], 200) });
1108+
const lane = await carveLane("peaking", { start: "0", "playback-rate": "2", duration: "2" });
1109+
expect({
1110+
speaking: lane.at(0.8) <= lane.depth * 0.5 && lane.at(1.3) <= lane.depth * 0.5,
1111+
after: lane.at(2.5),
1112+
}).toEqual({ speaking: true, after: 0 });
1113+
});
1114+
1115+
it("ducks against the part of the bed that plays", async () => {
1116+
// The bed file is silent for 4 s and loud after; the clip starts 4 s in, so it is loud under the voice.
1117+
stubFiles({ "voice.wav": tone(4, [1, 3]), "bed.wav": tone(8, [4, 8], 200, 0.9) });
1118+
const lane = await carveLane(
1119+
"gain",
1120+
{ start: "0" },
1121+
{ "media-start": "4", duration: "4" },
1122+
1,
1123+
);
1124+
expect(Math.min(...lane.points.map((p) => p.v))).toBeLessThan(0);
1125+
});
1126+
});
9921127
});
9931128

9941129
describe("AudioFxGroup successive edits", () => {

‎packages/studio/src/components/editor/useFxCarve.ts‎

Lines changed: 11 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ import { automationAttrValue, HF_AUDIO_AUTOMATION_ATTR } from "./propertyPanelAu
3333
import { trackCarveChanged } from "./audioFxTelemetry.js";
3434
import type { DomEditSelection } from "./domEditingTypes";
3535
import { usePlayerStore } from "../../player";
36-
import { carveLanes, measureCarve, mintCarveNodes } from "./useFxCarveNodes.js";
36+
import { carveLanes, measureCarve, mintCarveNodes, type CarveClip } from "./useFxCarveNodes.js";
37+
import { readClipClock } from "./clipAudioClock.js";
3738
import { spanOf } from "./propertyPanelAudioFxGroupUtils.js";
3839
import type { AudioTrackOption } from "./propertyPanelFxCarveModule.js";
3940

@@ -93,11 +94,8 @@ function carveNeedsReanalysis(
9394
* what lets the src and the start be non-null by construction downstream
9495
* instead of by assertion.
9596
*/
96-
function resolveCarveVoices(
97-
doc: Document,
98-
sources: readonly string[],
99-
): { src: string; start: string | null }[] {
100-
const voices: { src: string; start: string | null }[] = [];
97+
function resolveCarveVoices(doc: Document, sources: readonly string[]): CarveClip[] {
98+
const voices: CarveClip[] = [];
10199
for (const id of sources) {
102100
const el = doc.getElementById(id);
103101
// By tag name, not `instanceof HTMLAudioElement`: these elements belong to
@@ -106,7 +104,8 @@ function resolveCarveVoices(
106104
if (el?.tagName !== "AUDIO") continue;
107105
const src = el.getAttribute("src");
108106
if (!src) continue;
109-
voices.push({ src, start: el.getAttribute("data-start") });
107+
const clock = readClipClock((name) => el.getAttribute(name));
108+
voices.push({ src, start: el.getAttribute("data-start"), clock });
110109
}
111110
return voices;
112111
}
@@ -363,14 +362,11 @@ export function useFxCarve(
363362
if (voices.length === 0) return;
364363
setAnalysing(true);
365364
try {
366-
const bedSrc = element.element?.getAttribute("src");
367-
const measured = await measureCarve(
368-
doc,
369-
voices,
370-
active.strength,
371-
element.dataAttributes?.["start"],
372-
bedSrc,
373-
);
365+
const measured = await measureCarve(doc, voices, active.strength, {
366+
src: element.element?.getAttribute("src"),
367+
start: element.dataAttributes?.["start"],
368+
clock: readClipClock((name) => element.dataAttributes?.[name.slice(5)]),
369+
});
374370
if (!measured) return;
375371
await persistMeasuredCarve(measured);
376372
} catch {

0 commit comments

Comments
 (0)