Skip to content

Commit 8e4ce74

Browse files
perf(engine): long renders encode their audio in parallel segments (#5788)
* perf(engine): long renders encode their audio in parallel segments * fix(engine): parallel audio encode counts packets from whole samples The expected packet count came from a floating-point duration, so frame-aligned lengths such as 147.52 s (one ulp above 7080960 samples) expected one packet more than the encoder writes, the last segment check threw, and the render lost its audio. Round to whole samples first; a 147.52 s test fails without the fix.
1 parent aba5eae commit 8e4ce74

3 files changed

Lines changed: 292 additions & 23 deletions

File tree

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
import { spawnSync } from "node:child_process";
2+
import { mkdtempSync, rmSync } from "node:fs";
3+
import { tmpdir } from "node:os";
4+
import { join } from "node:path";
5+
import { afterAll, describe, expect, it } from "vitest";
6+
import { getFfmpegBinary, getFfprobeBinary } from "../utils/ffmpegBinaries.js";
7+
import { aacSegmentCount, encodeAac, splitAdtsFrames } from "./aacEncode.js";
8+
9+
const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0;
10+
11+
function ffmpeg(args: string[]): Buffer {
12+
const result = spawnSync(getFfmpegBinary(), ["-nostdin", "-loglevel", "error", ...args], {
13+
maxBuffer: 1 << 28,
14+
});
15+
if (result.status !== 0) throw new Error(`ffmpeg failed: ${result.stderr?.toString()}`);
16+
return result.stdout;
17+
}
18+
19+
function decode(path: string, filter = "anull"): Float32Array {
20+
const pcm = ffmpeg(["-i", path, "-af", filter, "-f", "f32le", "-ac", "2", "-"]);
21+
return new Float32Array(pcm.buffer, pcm.byteOffset, pcm.length / 4);
22+
}
23+
24+
/** Where the audio stream says it ends, in 1/48000 s ticks. */
25+
function streamEnd(path: string): string {
26+
const args = ["-v", "error", "-select_streams", "a", "-show_entries", "stream=duration_ts"];
27+
const result = spawnSync(getFfprobeBinary(), [...args, "-of", "csv=p=0", "--", path], {
28+
encoding: "utf-8",
29+
});
30+
return result.stdout.trim();
31+
}
32+
33+
/** Join k decodes from segment k-1's last packet and segment k's first: samples [(cut-1)*1024, cut*1024). */
34+
function joinCentres(seconds: number, segments: number): number[] {
35+
const packets = Math.ceil(Math.round(seconds * 48_000) / 1024) + 1;
36+
return Array.from({ length: segments - 1 }, (_, k) => {
37+
const cut = Math.round(((k + 1) * packets) / segments);
38+
return (cut - 1) * 1024 + 512;
39+
});
40+
}
41+
42+
/** Error against the source: overall RMS, and the peak within 4096 frames of each cut versus elsewhere. */
43+
function errorStats(source: Float32Array, coded: Float32Array, cutFrames: number[]) {
44+
const nearCut = (i: number) =>
45+
cutFrames.some((cut) => Math.abs(Math.floor(i / 2) - cut * 1024) < 4096);
46+
let sum = 0;
47+
let peakNearCut = 0;
48+
let peakElsewhere = 0;
49+
for (let i = 0; i < source.length; i++) {
50+
const error = Math.abs(coded[i]! - source[i]!);
51+
sum += error * error;
52+
if (nearCut(i)) peakNearCut = Math.max(peakNearCut, error);
53+
else peakElsewhere = Math.max(peakElsewhere, error);
54+
}
55+
return { rms: Math.sqrt(sum / source.length), peakNearCut, peakElsewhere };
56+
}
57+
58+
describe("aacSegmentCount", () => {
59+
it("keeps audio under two minutes in one encoder", () => {
60+
expect(aacSegmentCount(119)).toBe(1);
61+
});
62+
63+
it("gives each segment at least a minute, up to eight segments", () => {
64+
expect(aacSegmentCount(180)).toBe(3);
65+
expect(aacSegmentCount(1800)).toBe(8);
66+
});
67+
});
68+
69+
describe.skipIf(!HAS_FFMPEG)("encodeAac", () => {
70+
const dir = mkdtempSync(join(tmpdir(), "hf-aac-encode-"));
71+
afterAll(() => rmSync(dir, { recursive: true, force: true }));
72+
73+
it("encodes a long mix in parallel segments that decode like one encoder's output", async () => {
74+
const source = join(dir, "mix.wav");
75+
ffmpeg([
76+
...["-f", "lavfi", "-i", "anoisesrc=d=150.5:c=pink:r=48000:a=0.2"],
77+
...["-f", "lavfi", "-i", "sine=f=330:d=150.5:r=48000"],
78+
...["-filter_complex", "[0][1]amix=2:normalize=0,aformat=channel_layouts=stereo"],
79+
...["-c:a", "pcm_f32le", "-y", source],
80+
]);
81+
const single = join(dir, "single.m4a");
82+
const split = join(dir, "split.m4a");
83+
ffmpeg(["-i", source, "-t", "150.5", "-acodec", "aac", "-b:a", "192k", "-y", single]);
84+
expect((await encodeAac(source, split, 150.5, { timeout: 120_000 })).success).toBe(true);
85+
// 150.5 s is not a whole number of AAC frames, so the last packet is short.
86+
expect(streamEnd(split)).toBe(streamEnd(single));
87+
88+
const packets = (path: string) =>
89+
splitAdtsFrames(ffmpeg(["-i", path, "-c", "copy", "-f", "adts", "-"]));
90+
const singlePackets = packets(single);
91+
const splitPackets = packets(split);
92+
expect(splitPackets.length).toBe(singlePackets.length);
93+
expect(Buffer.concat(splitPackets).equals(Buffer.concat(singlePackets))).toBe(false);
94+
95+
const reference = decode(source);
96+
const singleDecoded = decode(single);
97+
const splitDecoded = decode(split);
98+
expect(splitDecoded.length).toBe(singleDecoded.length);
99+
const cuts = [Math.round(singlePackets.length / 2)];
100+
const singleError = errorStats(reference, singleDecoded, cuts);
101+
const splitError = errorStats(reference, splitDecoded, cuts);
102+
// Measured: under 1% more RMS error, and cut peaks at one encoder's own level.
103+
expect(splitError.rms).toBeLessThan(singleError.rms * 1.02);
104+
expect(splitError.peakNearCut).toBeLessThan(singleError.peakElsewhere);
105+
}, 120_000);
106+
107+
it("encodes a frame-aligned duration whose sample count is not exact in floating point", async () => {
108+
// 147.52 s is exactly 6915 AAC frames, but 147.52 * 48000 lands one ulp above it.
109+
const source = join(dir, "aligned.wav");
110+
ffmpeg(["-f", "lavfi", "-i", "sine=f=440:d=147.52:r=48000", "-ac", "2", "-y", source]);
111+
const single = join(dir, "aligned-single.m4a");
112+
const split = join(dir, "aligned-split.m4a");
113+
ffmpeg(["-i", source, "-t", "147.52", "-acodec", "aac", "-b:a", "192k", "-y", single]);
114+
expect((await encodeAac(source, split, 147.52, { timeout: 120_000 })).success).toBe(true);
115+
expect(streamEnd(split)).toBe(streamEnd(single));
116+
}, 120_000);
117+
118+
it("leaves no click at the joins of a pure tone", async () => {
119+
const source = join(dir, "tone.wav");
120+
ffmpeg([
121+
...["-f", "lavfi", "-i", "sine=f=440:d=150.5:r=48000"],
122+
...["-af", "volume=0.5,aformat=channel_layouts=stereo", "-c:a", "pcm_f32le", "-y", source],
123+
]);
124+
const output = join(dir, "tone.m4a");
125+
expect((await encodeAac(source, output, 150.5, { timeout: 120_000 })).success).toBe(true);
126+
127+
const joins = joinCentres(150.5, aacSegmentCount(150.5));
128+
expect(joins).toHaveLength(1);
129+
const window = 2 * 2400; // 50 ms of interleaved stereo
130+
const peakAt = (signal: Float32Array, frame: number) => {
131+
let peak = 0;
132+
for (let i = Math.max(0, frame * 2 - window / 2); i < frame * 2 + window / 2; i++) {
133+
peak = Math.max(peak, Math.abs(signal[i] ?? 0));
134+
}
135+
return peak;
136+
};
137+
const medianPeak = (signal: Float32Array) => {
138+
const peaks = Array.from({ length: Math.floor(signal.length / window) }, (_, w) =>
139+
peakAt(signal, w * (window / 2) + window / 4),
140+
).sort((a, b) => a - b);
141+
return peaks[Math.floor(peaks.length / 2)]!;
142+
};
143+
// A 440 Hz tone has almost nothing above 2 kHz, so a click shows as a broadband spike there;
144+
// a dropped or repeated frame shows as a phase jump against the source.
145+
const highPassed = decode(output, Array(4).fill("highpass=f=2000").join(","));
146+
const reference = decode(source);
147+
const residual = decode(output).map((sample, i) => sample - (reference[i] ?? 0));
148+
const [hpMedian, residualMedian] = [medianPeak(highPassed), medianPeak(residual)];
149+
// Measured: joins sit at the median; the source's own stream-copy joins read about 40 dB above it.
150+
for (const centre of joins) {
151+
expect(peakAt(highPassed, centre)).toBeLessThan(hpMedian * 2);
152+
expect(peakAt(residual, centre)).toBeLessThan(residualMedian * 2);
153+
}
154+
}, 120_000);
155+
});
Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
/**
2+
* AAC for the mixed 48 kHz audio. One encoder is single-threaded (about 60x real time),
3+
* so long audio is encoded as frame-aligned segments in parallel and their frames joined.
4+
*/
5+
6+
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
7+
import { dirname, join } from "node:path";
8+
import { runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
9+
10+
const SAMPLE_RATE = 48_000;
11+
const FRAME_SAMPLES = 1024;
12+
const MIN_SEGMENT_SECONDS = 60;
13+
// Fixed rather than the core count, so the bytes (and the planHash) do not depend on the machine.
14+
const MAX_SEGMENTS = 8;
15+
// Encoded on each side of a segment and dropped. The encoder's rate control takes about
16+
// 10 s to settle (its error is higher until then), so a segment starts 10 s before its cut.
17+
const PRE_ROLL_FRAMES = 469;
18+
const POST_ROLL_FRAMES = 8;
19+
export const AAC_ARGS = ["-acodec", "aac", "-b:a", "192k"];
20+
21+
export function aacSegmentCount(totalDuration: number): number {
22+
return Math.max(1, Math.min(MAX_SEGMENTS, Math.floor(totalDuration / MIN_SEGMENT_SECONDS)));
23+
}
24+
25+
export function splitAdtsFrames(bytes: Buffer): Buffer[] {
26+
const frames: Buffer[] = [];
27+
for (let at = 0; at < bytes.length; ) {
28+
const length = ((bytes[at + 3]! & 0x03) << 11) | (bytes[at + 4]! << 3) | (bytes[at + 5]! >> 5);
29+
if (bytes[at] !== 0xff || (bytes[at + 1]! & 0xf0) !== 0xf0 || length < 7) {
30+
throw new Error(`Not an ADTS frame at byte ${at}`);
31+
}
32+
frames.push(bytes.subarray(at, at + length));
33+
at += length;
34+
}
35+
return frames;
36+
}
37+
38+
/**
39+
* Packet g of the stream covers samples [(g-1)*1024, (g+1)*1024); packet 0 is the
40+
* encoder delay. Segment k encodes from `firstFrame` and keeps its packets [keepFrom, keepTo).
41+
*/
42+
function planSegments(totalDuration: number, count: number) {
43+
// Round to whole samples first: 147.52 * 48000 is 7080960.000000001 in floating point.
44+
const packets = Math.ceil(Math.round(totalDuration * SAMPLE_RATE) / FRAME_SAMPLES) + 1;
45+
const cuts = Array.from({ length: count + 1 }, (_, k) => Math.round((k * packets) / count));
46+
return cuts.slice(0, -1).map((cut, k) => {
47+
const next = cuts[k + 1]!;
48+
const firstFrame = Math.max(0, cut - PRE_ROLL_FRAMES);
49+
const end = k === count - 1 ? "" : `:end_sample=${(next + POST_ROLL_FRAMES) * FRAME_SAMPLES}`;
50+
const trim = `atrim=start_sample=${firstFrame * FRAME_SAMPLES}${end}`;
51+
return { trim, keepFrom: cut - firstFrame, keepTo: next - firstFrame };
52+
});
53+
}
54+
55+
/** `input` is a 48 kHz WAV of exactly `totalDuration`. */
56+
export async function encodeAac(
57+
input: string,
58+
outputPath: string,
59+
totalDuration: number,
60+
{ signal, timeout }: { signal?: AbortSignal; timeout: number },
61+
): Promise<RunFfmpegResult> {
62+
const count = aacSegmentCount(totalDuration);
63+
if (count === 1) {
64+
const args = ["-i", input, "-t", String(totalDuration), ...AAC_ARGS, "-y", outputPath];
65+
return runFfmpeg(args, { signal, timeout });
66+
}
67+
68+
const dir = mkdtempSync(join(dirname(outputPath), ".aac-segments-"));
69+
try {
70+
const segments = planSegments(totalDuration, count).map((segment, k) => ({
71+
...segment,
72+
path: join(dir, `${k}.aac`),
73+
}));
74+
const results = await Promise.all(
75+
segments.map(({ trim, path }) =>
76+
runFfmpeg(["-i", input, "-af", trim, ...AAC_ARGS, "-f", "adts", "-y", path], {
77+
signal,
78+
timeout,
79+
}),
80+
),
81+
);
82+
const failed = results.find((result) => !result.success);
83+
if (failed) return failed;
84+
85+
const joined = join(dir, "joined.aac");
86+
const frames = segments.flatMap(({ path, keepFrom, keepTo }) => {
87+
const segmentFrames = splitAdtsFrames(readFileSync(path));
88+
if (segmentFrames.length < keepTo) {
89+
throw new Error(`AAC segment ${path} has ${segmentFrames.length} frames, needs ${keepTo}`);
90+
}
91+
return segmentFrames.slice(keepFrom, keepTo);
92+
});
93+
writeFileSync(joined, Buffer.concat(frames));
94+
// ADTS carries neither the encoder delay nor the last packet's short duration: the offset
95+
// restores the delay, and setts ends the stream at totalDuration as one encoder would.
96+
const offset = String(-FRAME_SAMPLES / SAMPLE_RATE);
97+
const end = `setts=duration='if(gte((PTS+DURATION)*TB,${totalDuration}),round(${totalDuration}/TB-PTS),DURATION)'`;
98+
const muxed = await runFfmpeg(
99+
[
100+
"-itsoffset",
101+
offset,
102+
"-f",
103+
"aac",
104+
"-i",
105+
joined,
106+
"-c",
107+
"copy",
108+
"-bsf:a",
109+
end,
110+
"-y",
111+
outputPath,
112+
],
113+
{ signal, timeout },
114+
);
115+
const slowest = Math.max(...results.map((result) => result.durationMs));
116+
return { ...muxed, durationMs: slowest + muxed.durationMs };
117+
} finally {
118+
rmSync(dir, { recursive: true, force: true });
119+
}
120+
}

‎packages/engine/src/services/audioMixer.ts‎

Lines changed: 17 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
} from "../utils/urlDownloader.js";
1616
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
1717
import { formatFfmpegError, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
18+
import { AAC_ARGS, aacSegmentCount, encodeAac } from "./aacEncode.js";
1819
import { audioSubprocessEvidence } from "./audioFailureDiagnostic.js";
1920
import { unwrapTemplate } from "../utils/htmlTemplate.js";
2021
import { resolveMediaElementSrc, resolveProjectRelativeSrc } from "./videoFrameExtractor.js";
@@ -827,7 +828,7 @@ async function mixAudioTracks(
827828
scriptPath,
828829
"-map",
829830
"[out]",
830-
...(floatWav ? ["-acodec", "pcm_f32le", "-ar", "48000"] : ["-acodec", "aac", "-b:a", "192k"]),
831+
...(floatWav ? ["-acodec", "pcm_f32le", "-ar", "48000"] : AAC_ARGS),
831832
"-t",
832833
String(totalDuration),
833834
"-y",
@@ -1059,12 +1060,12 @@ async function mixGroupMembers(
10591060
}
10601061

10611062
/**
1062-
* Sum the tracks to a float WAV, run the master chain over the sum, then AAC-encode.
1063+
* Sum the tracks to a float WAV, run the master chain (if any) over the sum, then AAC-encode.
10631064
* Groups are already folded into `tracks`, and the sum stays float until the encoder.
10641065
*/
1065-
async function mixThroughMasterBus(
1066+
async function mixThroughFloatSum(
10661067
tracks: AudioTrack[],
1067-
masterChain: HfAudioFxChain,
1068+
masterChain: HfAudioFxChain | undefined,
10681069
outputPath: string,
10691070
workDir: string,
10701071
totalDuration: number,
@@ -1075,25 +1076,18 @@ async function mixThroughMasterBus(
10751076
const sum = await mixAudioTracks(tracks, sumPath, totalDuration, signal, config, true);
10761077
if (!sum.success) return { ...sum, outputPath };
10771078

1078-
const fx = await applyAudioFxChain(sumPath, masterChain, join(workDir, "master-fx.wav"), {
1079-
trackId: "master",
1079+
const encoderInput = masterChain
1080+
? (
1081+
await applyAudioFxChain(sumPath, masterChain, join(workDir, "master-fx.wav"), {
1082+
trackId: "master",
1083+
signal,
1084+
})
1085+
).path
1086+
: sumPath;
1087+
const encoded = await encodeAac(encoderInput, outputPath, totalDuration, {
10801088
signal,
1089+
timeout: config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout,
10811090
});
1082-
const encoded = await runFfmpeg(
1083-
[
1084-
"-i",
1085-
fx.path,
1086-
"-t",
1087-
String(totalDuration),
1088-
"-acodec",
1089-
"aac",
1090-
"-b:a",
1091-
"192k",
1092-
"-y",
1093-
outputPath,
1094-
],
1095-
{ signal, timeout: config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout },
1096-
);
10971091
if (!encoded.success) {
10981092
const failure = ffmpegFailure("mix", encoded);
10991093
return {
@@ -1613,8 +1607,8 @@ export async function processCompositionAudio(
16131607
let mixResult: MixResult;
16141608
try {
16151609
mixResult =
1616-
masterChain && tracks.length > 0
1617-
? await mixThroughMasterBus(
1610+
(masterChain || aacSegmentCount(totalDuration) > 1) && tracks.length > 0
1611+
? await mixThroughFloatSum(
16181612
tracks,
16191613
masterChain,
16201614
outputPath,

0 commit comments

Comments
 (0)