Skip to content

Commit bb12223

Browse files
committed
fix(engine): mix any number of audio clips by summing them in stages
One ffmpeg graph held every clip of a group (or of the whole film) as its own input, so a long project ran into the open-file and input limits and the export failed. Past 96 clips the mixer now folds slices into one running float sum, with no cap on the total; mixes of 96 clips or fewer take the same single-graph path as before. A mix process now also gets a deadline that grows with the film length, and a failed ffmpeg run keeps how long it ran and the last lines of its stderr.
1 parent 3badc9b commit bb12223

3 files changed

Lines changed: 437 additions & 22 deletions

File tree

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
import { spawnSync } from "node:child_process";
2+
import { copyFileSync, mkdtempSync, readdirSync, rmSync } from "node:fs";
3+
import { tmpdir } from "node:os";
4+
import { dirname, join } from "node:path";
5+
import { afterEach, describe, expect, it } from "vitest";
6+
import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
7+
import { MAX_MIX_INPUTS, MIXED_AUDIO_FILENAME, processCompositionAudio } from "./audioMixer.js";
8+
9+
const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0;
10+
const tempDirs: string[] = [];
11+
12+
function meanVolumeDb(path: string): number {
13+
const result = spawnSync(
14+
getFfmpegBinary(),
15+
["-nostdin", "-hide_banner", "-i", path, "-af", "volumedetect", "-f", "null", "-"],
16+
{ encoding: "utf-8" },
17+
);
18+
const match = result.stderr.match(/mean_volume:\s*(-?[\d.]+) dB/);
19+
if (result.status !== 0 || !match?.[1]) throw new Error(`Could not measure: ${result.stderr}`);
20+
return Number(match[1]);
21+
}
22+
23+
function peakDb(path: string): number {
24+
const result = spawnSync(
25+
getFfmpegBinary(),
26+
["-nostdin", "-hide_banner", "-i", path, "-af", "astats", "-f", "null", "-"],
27+
{ encoding: "utf-8" },
28+
);
29+
const match = result.stderr.match(/Peak level dB:\s*(-?[\d.]+)/);
30+
if (result.status !== 0 || !match?.[1]) throw new Error(`Could not measure: ${result.stderr}`);
31+
return Number(match[1]);
32+
}
33+
34+
function writeTone(path: string, seconds: number, gain: number): void {
35+
const result = spawnSync(
36+
getFfmpegBinary(),
37+
[
38+
"-nostdin",
39+
"-v",
40+
"error",
41+
"-f",
42+
"lavfi",
43+
"-i",
44+
`sine=frequency=440:duration=${seconds}:sample_rate=48000`,
45+
"-af",
46+
`volume=${gain}`,
47+
"-c:a",
48+
"pcm_s16le",
49+
path,
50+
],
51+
{ encoding: "utf-8" },
52+
);
53+
if (result.status !== 0) throw new Error(`Could not write tone: ${result.stderr}`);
54+
}
55+
56+
const clip = (i: number, start: number, end: number, groupId?: string) => ({
57+
id: `c${i}`,
58+
src: `a${i}.wav`,
59+
start,
60+
end,
61+
mediaStart: 0,
62+
layer: i,
63+
volume: 1,
64+
type: "audio" as const,
65+
...(groupId ? { groupId } : {}),
66+
});
67+
68+
describe.skipIf(!HAS_FFMPEG)("a mix with more clips than one ffmpeg graph can hold", () => {
69+
afterEach(() => {
70+
for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
71+
});
72+
73+
async function mix(
74+
count: number,
75+
groupId: string | undefined,
76+
shape: { stackGain: number } | "sequence",
77+
) {
78+
const projectDir = mkdtempSync(join(tmpdir(), "hf-many-"));
79+
const workDir = mkdtempSync(join(tmpdir(), "hf-many-work-"));
80+
tempDirs.push(projectDir, workDir);
81+
const stacked = shape !== "sequence";
82+
writeTone(join(projectDir, "a.wav"), stacked ? 2 : 0.2, stacked ? shape.stackGain : 1);
83+
for (let i = 0; i < count; i++)
84+
copyFileSync(join(projectDir, "a.wav"), join(projectDir, `a${i}.wav`));
85+
const clips = Array.from({ length: count }, (_, i) =>
86+
stacked ? clip(i, 0, 2, groupId) : clip(i, i * 0.05, (i + 1) * 0.05, groupId),
87+
);
88+
const out = join(projectDir, MIXED_AUDIO_FILENAME);
89+
const sum = join(projectDir, "sum.wav");
90+
const result = await processCompositionAudio(
91+
clips,
92+
projectDir,
93+
workDir,
94+
out,
95+
stacked ? 2 : count * 0.05,
96+
undefined,
97+
{ keepEncoderInputAt: sum },
98+
);
99+
return { result, out, sum };
100+
}
101+
102+
it("mixes 200 clips in one group at the level of a short group", async () => {
103+
const small = await mix(8, "sfx", "sequence");
104+
const many = await mix(200, "sfx", "sequence");
105+
expect(many.result.failures ?? []).toEqual([]);
106+
expect(many.result.success).toBe(true);
107+
expect(Math.abs(meanVolumeDb(many.out) - meanVolumeDb(small.out))).toBeLessThan(1);
108+
}, 120_000);
109+
110+
it("mixes 200 ungrouped clips at the level of a short mix", async () => {
111+
const small = await mix(8, undefined, "sequence");
112+
const many = await mix(200, undefined, "sequence");
113+
expect(many.result.failures ?? []).toEqual([]);
114+
expect(many.result.success).toBe(true);
115+
expect(Math.abs(meanVolumeDb(many.out) - meanVolumeDb(small.out))).toBeLessThan(1);
116+
expect(readdirSync(dirname(many.out)).filter((f) => f.startsWith(".mix-batches-"))).toEqual([]);
117+
}, 120_000);
118+
119+
it.each([
120+
["ungrouped", undefined],
121+
["grouped", "sfx"],
122+
])(
123+
"sums %s clips past the slice size at the gain the flat mix gives",
124+
async (_name, groupId) => {
125+
// Stacked coherent clips add, so 120 at gain g and 60 at gain 2g have the same sum.
126+
// 60 is one flat graph; 120 is staged. The float sums must peak alike.
127+
const flat = await mix(60, groupId, { stackGain: 0.1 });
128+
const staged = await mix(MAX_MIX_INPUTS + 24, groupId, { stackGain: 0.05 });
129+
expect(flat.result.success).toBe(true);
130+
expect(staged.result.success).toBe(true);
131+
expect(Math.abs(peakDb(staged.sum) - peakDb(flat.sum))).toBeLessThan(0.1);
132+
},
133+
120_000,
134+
);
135+
});

‎packages/engine/src/services/audioMixer.test.ts‎

Lines changed: 149 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ vi.mock("../utils/ffprobe.js", async (importOriginal) => {
8383

8484
import { RATE_RANGE } from "@hyperframes/core/audio-automation";
8585
import { AudioFxRenderError } from "./audioFxRender.js";
86-
import { parseAudioElements, processCompositionAudio } from "./audioMixer.js";
86+
import { MAX_MIX_INPUTS, parseAudioElements, processCompositionAudio } from "./audioMixer.js";
8787
import { encodeAac } from "./aacEncode.js";
8888

8989
describe("parseAudioElements strict literal timing", () => {
@@ -1093,6 +1093,140 @@ describe("processCompositionAudio", () => {
10931093
expect(runFfmpegMock).toHaveBeenCalledTimes(2);
10941094
});
10951095

1096+
it("keeps how long a killed group mix ran and ffmpeg's last words", async () => {
1097+
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
1098+
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
1099+
tempDirs.push(baseDir, workDir);
1100+
writeFileSync(join(baseDir, "voice.wav"), "stub");
1101+
1102+
const streamLines = Array.from(
1103+
{ length: 400 },
1104+
(_, i) =>
1105+
` Stream #${i}:0 -> #0:${i} (pcm_s16le (native) -> atrim (graph 0 input from stream ${i}:0))`,
1106+
);
1107+
runFfmpegMock
1108+
.mockResolvedValueOnce({
1109+
success: true,
1110+
durationMs: 1,
1111+
stderr: "",
1112+
exitCode: 0,
1113+
terminationReason: "exit",
1114+
})
1115+
.mockResolvedValueOnce({
1116+
success: false,
1117+
durationMs: 530_000,
1118+
stderr: [...streamLines, "Exiting normally, received signal 15."].join("\n"),
1119+
exitCode: 255,
1120+
signal: null,
1121+
terminationReason: "exit",
1122+
failureReason: "external_interruption",
1123+
});
1124+
1125+
const result = await processCompositionAudio(
1126+
[
1127+
{
1128+
id: "voice",
1129+
src: "voice.wav",
1130+
start: 0,
1131+
end: 2,
1132+
mediaStart: 0,
1133+
layer: 0,
1134+
volume: 1,
1135+
type: "audio",
1136+
groupId: "sfx",
1137+
},
1138+
],
1139+
baseDir,
1140+
workDir,
1141+
join(baseDir, "out.m4a"),
1142+
2,
1143+
);
1144+
1145+
const failure = result.failures?.find((f) => f.elementId === "sfx");
1146+
expect(failure?.reason).toBe("external_interruption");
1147+
expect(failure?.subprocess).toMatchObject({ exitCode: 255 });
1148+
expect(failure?.detail).toContain("ran 530000 ms");
1149+
expect(failure?.detail).toContain("Exiting normally, received signal 15.");
1150+
});
1151+
1152+
it.each([
1153+
["ungrouped", undefined],
1154+
["grouped", "sfx"],
1155+
])(
1156+
"never hands one ffmpeg more inputs than a graph holds, %s",
1157+
async (_name, groupId) => {
1158+
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
1159+
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
1160+
tempDirs.push(baseDir, workDir);
1161+
const trackCount = 2_000;
1162+
const elements = Array.from({ length: trackCount }, (_, i) => {
1163+
writeFileSync(join(baseDir, `clip-${i}.wav`), "stub");
1164+
return {
1165+
id: `clip-${i}`,
1166+
src: `clip-${i}.wav`,
1167+
start: i * 0.1,
1168+
end: i * 0.1 + 0.5,
1169+
mediaStart: 0,
1170+
layer: i,
1171+
volume: 1,
1172+
type: "audio" as const,
1173+
...(groupId ? { groupId } : {}),
1174+
};
1175+
});
1176+
runFfmpegMock.mockClear();
1177+
1178+
const result = await processCompositionAudio(
1179+
elements,
1180+
baseDir,
1181+
workDir,
1182+
join(baseDir, "out.m4a"),
1183+
trackCount * 0.1 + 0.5,
1184+
);
1185+
1186+
// The mocked ffmpeg writes no files, so only the ungrouped mix runs to the end.
1187+
if (!groupId) expect(result.success).toBe(true);
1188+
const inputsPerCall = runFfmpegMock.mock.calls.map(
1189+
(call) => (call[0] as string[]).filter((arg) => arg === "-i").length,
1190+
);
1191+
expect(Math.max(...inputsPerCall)).toBeLessThanOrEqual(MAX_MIX_INPUTS);
1192+
},
1193+
60_000,
1194+
);
1195+
1196+
it("lets a mix's ffmpeg deadline grow with the film length", async () => {
1197+
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
1198+
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
1199+
tempDirs.push(baseDir, workDir);
1200+
writeFileSync(join(baseDir, "voice.wav"), "stub");
1201+
const voice = {
1202+
id: "voice",
1203+
src: "voice.wav",
1204+
start: 0,
1205+
end: 2,
1206+
mediaStart: 0,
1207+
layer: 0,
1208+
volume: 1,
1209+
type: "audio" as const,
1210+
};
1211+
const mixTimeout = async (totalDuration: number) => {
1212+
runFfmpegMock.mockClear();
1213+
await processCompositionAudio(
1214+
[voice],
1215+
baseDir,
1216+
workDir,
1217+
join(baseDir, "out.m4a"),
1218+
totalDuration,
1219+
);
1220+
const mixCall = runFfmpegMock.mock.calls.find((call) =>
1221+
(call[0] as string[]).includes("-filter_complex_script"),
1222+
);
1223+
return (mixCall?.[1] as { timeout?: number } | undefined)?.timeout;
1224+
};
1225+
1226+
expect(await mixTimeout(60)).toBe(300_000);
1227+
expect(await mixTimeout(36_000)).toBe(3_600_000);
1228+
});
1229+
10961230
it("preserves and classifies unsupported FFmpeg filter failures", async () => {
10971231
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
10981232
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
@@ -1466,21 +1600,22 @@ describe("processCompositionAudio", () => {
14661600
expect(result.success).toBe(true);
14671601
expect(result.tracksProcessed).toBe(trackCount);
14681602

1469-
const mixArgs = runFfmpegMock.mock.calls.at(-1)?.[0] as string[];
1470-
expect(mixArgs).toContain("-filter_complex_script");
1471-
expect(mixArgs).not.toContain("-filter_complex");
1472-
1473-
// The only things that scale with track count are the -i pairs (short,
1474-
// fixed-size each) and the filter SCRIPT FILE's content (off the command
1475-
// line entirely) — not the args array's own total character length.
1476-
const argsLength = mixArgs.join(" ").length;
1477-
expect(argsLength).toBeLessThan(20_000);
1603+
// 150 tracks exceed one graph: two slice sums, the fold of those two, then the final mix.
1604+
const mixCalls = runFfmpegMock.mock.calls.slice(-4).map((call) => call[0] as string[]);
1605+
for (const mixArgs of mixCalls) {
1606+
expect(mixArgs).toContain("-filter_complex_script");
1607+
expect(mixArgs).not.toContain("-filter_complex");
1608+
// Only the -i pairs and the filter script's content scale with track count.
1609+
expect(mixArgs.join(" ").length).toBeLessThan(20_000);
1610+
}
14781611

1479-
const filter = capturedFilterScripts.at(-1);
1480-
expect(filter).toContain(`amix=inputs=${trackCount}`);
1612+
const [first, second, folded, last] = capturedFilterScripts.slice(-4);
1613+
expect(first).toContain(`amix=inputs=${MAX_MIX_INPUTS}`);
1614+
expect(second).toContain(`amix=inputs=${trackCount - MAX_MIX_INPUTS}`);
1615+
expect(folded).toContain("amix=inputs=2");
1616+
expect(last).toContain("amix=inputs=1");
14811617
// Each track is trimmed to its authored clip; only the mixed output is padded.
1482-
expect((filter?.match(/atrim=/g) ?? []).length).toBe(trackCount + 1);
1483-
expect((filter?.match(/apad,/g) ?? []).length).toBe(1);
1618+
expect((last?.match(/apad,/g) ?? []).length).toBe(1);
14841619
// 150 real files + 150 existence checks: ~60ms on Linux, but well past the 5s
14851620
// default on the Windows lane, where each file create is orders slower.
14861621
}, 30_000);

0 commit comments

Comments
 (0)