Skip to content

Commit 37b8868

Browse files
authored
fix(audio): preserve causes and use portable padding (#2769)
* fix(audio): preserve causes and use portable padding * fix(audio): address failure taxonomy review
1 parent 72e2f08 commit 37b8868

18 files changed

Lines changed: 839 additions & 84 deletions

‎packages/engine/src/index.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,8 +198,12 @@ export {
198198
export { createVideoFrameInjector } from "./services/videoFrameInjector.js";
199199

200200
export { parseAudioElements, processCompositionAudio } from "./services/audioMixer.js";
201+
export { cloneCaptureWarning, cloneCaptureWarnings } from "./services/captureWarning.js";
201202
export type {
202203
AudioElement,
204+
AudioFailureReason,
205+
AudioFailureStage,
206+
AudioProcessingFailure,
203207
AudioTrack,
204208
AudioVolumeKeyframe,
205209
MixResult,

‎packages/engine/src/services/audioMixer.test.ts‎

Lines changed: 278 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
// fallow-ignore-file code-duplication
12
import { afterEach, describe, expect, it, vi } from "vitest";
23
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
34
import { join } from "node:path";
@@ -10,10 +11,16 @@ import { tmpdir } from "node:os";
1011
// filter content synchronously, while the file still exists, into an
1112
// index-aligned side array (rather than re-reading it from disk after
1213
// processCompositionAudio resolves, by which point it's already gone).
13-
const { runFfmpegMock, capturedFilterScripts } = vi.hoisted(() => {
14+
const { runFfmpegMock, capturedFilterScripts, extractAudioMetadataMock } = vi.hoisted(() => {
1415
const capturedFilterScripts: string[] = [];
1516
return {
1617
capturedFilterScripts,
18+
extractAudioMetadataMock: vi.fn(async () => ({
19+
durationSeconds: 2,
20+
sampleRate: 48_000,
21+
channels: 2,
22+
audioCodec: "aac",
23+
})),
1724
runFfmpegMock: vi.fn(async (args: string[]) => {
1825
const legacyIdx = args.indexOf("-filter_complex_script");
1926
const currentIdx = args.indexOf("-/filter_complex");
@@ -29,9 +36,15 @@ const { runFfmpegMock, capturedFilterScripts } = vi.hoisted(() => {
2936
};
3037
});
3138

32-
vi.mock("../utils/runFfmpeg.js", () => ({
33-
runFfmpeg: runFfmpegMock,
34-
}));
39+
vi.mock("../utils/runFfmpeg.js", async (importOriginal) => {
40+
const actual = await importOriginal<typeof import("../utils/runFfmpeg.js")>();
41+
return { ...actual, runFfmpeg: runFfmpegMock };
42+
});
43+
44+
vi.mock("../utils/ffprobe.js", async (importOriginal) => {
45+
const actual = await importOriginal<typeof import("../utils/ffprobe.js")>();
46+
return { ...actual, extractAudioMetadata: extractAudioMetadataMock };
47+
});
3548

3649
import { parseAudioElements, processCompositionAudio } from "./audioMixer.js";
3750

@@ -40,12 +53,68 @@ describe("processCompositionAudio", () => {
4053

4154
afterEach(() => {
4255
runFfmpegMock.mockClear();
56+
extractAudioMetadataMock.mockReset();
57+
extractAudioMetadataMock.mockResolvedValue({
58+
durationSeconds: 2,
59+
sampleRate: 48_000,
60+
channels: 2,
61+
audioCodec: "aac",
62+
});
4363
capturedFilterScripts.length = 0;
4464
for (const dir of tempDirs.splice(0)) {
4565
rmSync(dir, { recursive: true, force: true });
4666
}
4767
});
4868

69+
it.each([
70+
{
71+
message: "AbortError: ffprobe operation aborted",
72+
reason: "cancelled",
73+
owner: "user",
74+
retryable: false,
75+
},
76+
{
77+
message: "ffprobe timed out after inactivity deadline",
78+
reason: "ffmpeg_timeout",
79+
owner: "system",
80+
retryable: true,
81+
},
82+
] as const)("classifies probe failure '$reason' independently", async (expected) => {
83+
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
84+
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
85+
tempDirs.push(baseDir, workDir);
86+
writeFileSync(join(baseDir, "voice.wav"), "stub");
87+
extractAudioMetadataMock.mockRejectedValueOnce(new Error(expected.message));
88+
89+
const result = await processCompositionAudio(
90+
[
91+
{
92+
id: "voice",
93+
src: "voice.wav",
94+
start: 0,
95+
end: 0,
96+
mediaStart: 0,
97+
layer: 0,
98+
volume: 1,
99+
type: "audio",
100+
},
101+
],
102+
baseDir,
103+
workDir,
104+
join(baseDir, "out.m4a"),
105+
2,
106+
);
107+
108+
expect(result.failures).toEqual([
109+
expect.objectContaining({
110+
stage: "probe",
111+
reason: expected.reason,
112+
owner: expected.owner,
113+
retryable: expected.retryable,
114+
}),
115+
]);
116+
});
117+
49118
it("preserves muted tracks and uses unity master gain by default", async () => {
50119
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
51120
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
@@ -79,6 +148,8 @@ describe("processCompositionAudio", () => {
79148

80149
expect(filter).toContain("volume=0");
81150
expect(filter).toContain("[mixed]volume=1[out]");
151+
expect(filter).toContain("apad,atrim=0:2");
152+
expect(filter).not.toContain("whole_dur");
82153
expect(filter).not.toContain("normalize=");
83154
expect(filter).not.toContain("weights=");
84155
});
@@ -157,7 +228,9 @@ describe("processCompositionAudio", () => {
157228
return {
158229
success: !isMissingCuePrepare,
159230
durationMs: 1,
160-
stderr: isMissingCuePrepare ? "Invalid data found when processing input" : "",
231+
stderr: isMissingCuePrepare
232+
? "https://media.example.test/private.wav?token=secret /tmp/hf/private secret.wav: Invalid data found when processing input"
233+
: "",
161234
exitCode: isMissingCuePrepare ? 1 : 0,
162235
};
163236
});
@@ -194,10 +267,205 @@ describe("processCompositionAudio", () => {
194267

195268
expect(result.success).toBe(false);
196269
expect(result.tracksProcessed).toBe(1);
197-
expect(result.error).toMatch(/Prepare failed: missing-cue/);
270+
expect(result.error).toContain("Invalid data found when processing input");
271+
expect(result.error).toContain("<redacted-url>");
272+
expect(result.error).toContain("<redacted-path>");
273+
expect(result.error).not.toContain("token=secret");
274+
expect(result.error).not.toContain("/tmp/hf/private");
275+
expect(result.error).not.toContain("secret.wav");
276+
expect(result.failures).toEqual([
277+
expect.objectContaining({
278+
stage: "prepare",
279+
reason: "invalid_media",
280+
owner: "user",
281+
retryable: false,
282+
elementId: "missing-cue",
283+
}),
284+
]);
198285
expect(runFfmpegMock).toHaveBeenCalledTimes(2);
199286
});
200287

288+
it("preserves and classifies unsupported FFmpeg filter failures", async () => {
289+
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
290+
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
291+
tempDirs.push(baseDir, workDir);
292+
writeFileSync(join(baseDir, "voice.wav"), "stub");
293+
294+
runFfmpegMock
295+
.mockResolvedValueOnce({
296+
success: true,
297+
durationMs: 1,
298+
stderr: "",
299+
exitCode: 0,
300+
terminationReason: "exit",
301+
})
302+
.mockResolvedValueOnce({
303+
success: false,
304+
durationMs: 1,
305+
stderr: "Error applying option 'whole_dur': Option not found",
306+
exitCode: 8,
307+
terminationReason: "exit",
308+
});
309+
310+
const result = await processCompositionAudio(
311+
[
312+
{
313+
id: "voice",
314+
src: "voice.wav",
315+
start: 0,
316+
end: 2,
317+
mediaStart: 0,
318+
layer: 0,
319+
volume: 1,
320+
type: "audio",
321+
},
322+
],
323+
baseDir,
324+
workDir,
325+
join(baseDir, "out.m4a"),
326+
2,
327+
);
328+
329+
expect(result.success).toBe(false);
330+
expect(result.error).toContain("Option not found");
331+
expect(result.failures).toEqual([
332+
expect.objectContaining({
333+
stage: "mix",
334+
reason: "ffmpeg_unsupported",
335+
owner: "system",
336+
retryable: false,
337+
}),
338+
]);
339+
});
340+
341+
it("preserves a sanitized FFmpeg spawn failure cause", async () => {
342+
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
343+
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
344+
tempDirs.push(baseDir, workDir);
345+
writeFileSync(join(baseDir, "voice.wav"), "stub");
346+
347+
runFfmpegMock.mockResolvedValueOnce({
348+
success: false,
349+
durationMs: 1,
350+
stderr: "",
351+
exitCode: null,
352+
terminationReason: "spawn_error",
353+
error: new Error("spawn C:\\private\\ffmpeg.exe ENOENT"),
354+
});
355+
356+
const result = await processCompositionAudio(
357+
[
358+
{
359+
id: "voice",
360+
src: "voice.wav",
361+
start: 0,
362+
end: 2,
363+
mediaStart: 0,
364+
layer: 0,
365+
volume: 1,
366+
type: "audio",
367+
},
368+
],
369+
baseDir,
370+
workDir,
371+
join(baseDir, "out.m4a"),
372+
2,
373+
);
374+
375+
expect(result.success).toBe(false);
376+
expect(result.error).toContain("ENOENT");
377+
expect(result.error).toContain("<redacted-path>");
378+
expect(result.error).not.toContain("C:\\private\\ffmpeg.exe");
379+
expect(result.failures).toEqual([
380+
expect.objectContaining({
381+
stage: "prepare",
382+
reason: "ffmpeg_unavailable",
383+
owner: "system",
384+
retryable: true,
385+
}),
386+
]);
387+
});
388+
389+
it("keeps invalid data from producer-generated mix inputs system-owned", async () => {
390+
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
391+
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
392+
tempDirs.push(baseDir, workDir);
393+
writeFileSync(join(baseDir, "voice.wav"), "stub");
394+
395+
runFfmpegMock
396+
.mockResolvedValueOnce({
397+
success: true,
398+
durationMs: 1,
399+
stderr: "",
400+
exitCode: 0,
401+
terminationReason: "exit",
402+
})
403+
.mockResolvedValueOnce({
404+
success: false,
405+
durationMs: 1,
406+
stderr: "Invalid data found when processing input",
407+
exitCode: 1,
408+
terminationReason: "exit",
409+
});
410+
411+
const result = await processCompositionAudio(
412+
[
413+
{
414+
id: "voice",
415+
src: "voice.wav",
416+
start: 0,
417+
end: 2,
418+
mediaStart: 0,
419+
layer: 0,
420+
volume: 1,
421+
type: "audio",
422+
},
423+
],
424+
baseDir,
425+
workDir,
426+
join(baseDir, "out.m4a"),
427+
2,
428+
);
429+
430+
expect(result.failures).toEqual([
431+
expect.objectContaining({
432+
stage: "mix",
433+
reason: "ffmpeg_failed",
434+
owner: "system",
435+
retryable: false,
436+
}),
437+
]);
438+
});
439+
440+
it("bounds per-cause details and the aggregate error across many authored IDs", async () => {
441+
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
442+
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
443+
tempDirs.push(baseDir, workDir);
444+
const oversizedId = "authored-id-".repeat(300);
445+
446+
const result = await processCompositionAudio(
447+
Array.from({ length: 3 }, (_, index) => ({
448+
id: `${oversizedId}-${index}`,
449+
src: `missing-${index}.wav`,
450+
start: 0,
451+
end: 2,
452+
mediaStart: 0,
453+
layer: index,
454+
volume: 1,
455+
type: "audio" as const,
456+
})),
457+
baseDir,
458+
workDir,
459+
join(baseDir, "out.m4a"),
460+
2,
461+
);
462+
463+
expect(result.success).toBe(false);
464+
expect(result.error?.length).toBeLessThanOrEqual(2_000);
465+
expect(result.failures).toHaveLength(3);
466+
expect(result.failures?.every((failure) => failure.detail.length <= 2_000)).toBe(true);
467+
});
468+
201469
it("uses frame-evaluated volume automation when keyframes are present", async () => {
202470
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
203471
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
@@ -402,7 +670,10 @@ describe("processCompositionAudio", () => {
402670

403671
const filter = capturedFilterScripts.at(-1);
404672
expect(filter).toContain(`amix=inputs=${trackCount}`);
405-
expect((filter?.match(/atrim=/g) ?? []).length).toBe(trackCount);
673+
// Each track is trimmed once to its authored clip and once after portable
674+
// indefinite `apad` to cap the padded stream at composition duration.
675+
expect((filter?.match(/atrim=/g) ?? []).length).toBe(trackCount * 2);
676+
expect((filter?.match(/apad,/g) ?? []).length).toBe(trackCount);
406677
});
407678

408679
it("retries with the current file-valued filter option when a nightly removes the legacy alias", async () => {

0 commit comments

Comments
 (0)