Skip to content

Commit b211fc7

Browse files
fix(engine): MP4, MOV and TS exports keep sound and picture in sync when streams start apart (#5455)
* fix(engine): exports place sound and picture right when a source's audio and video start apart * fix(engine): a late video stream plays to its end and distributed and HDR renders hide its lead * test(engine): pin a late video stream's trimmed start, held tail and HDR trim
1 parent ba08acb commit b211fc7

12 files changed

Lines changed: 443 additions & 161 deletions

File tree

‎packages/engine/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@ export {
213213
extractionFrameCountForDuration,
214214
resolveProjectRelativeSrc,
215215
getFrameAtTime,
216+
isVideoHiddenBeforeStreamStart,
216217
createFrameLookupTable,
217218
FrameLookupTable,
218219
analyzeClipMediaFit,

‎packages/engine/src/services/audioFxRender.ts‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -176,14 +176,7 @@ export function writeWav(
176176
writeFileSync(path, buf);
177177
}
178178

179-
/**
180-
* Split an interleaved buffer into one array per channel.
181-
*
182-
* The graph used to fold everything to mono, which collapsed a stereo bed's
183-
* width for the render only — and cost ~3 dB through the very mono-to-stereo
184-
* rematrix that `prepareAudioTrack`'s pan filter exists to avoid. Preview kept
185-
* the track stereo, so the two diverged the moment any effect was enabled.
186-
*/
179+
/** Split an interleaved buffer into one array per channel, keeping stereo width as preview does. */
187180
function deinterleave(samples: Float32Array, channels: number): Float32Array[] {
188181
if (channels <= 1) return [samples];
189182
const frames = Math.floor(samples.length / channels);

‎packages/engine/src/services/audioMixer.ts‎

Lines changed: 49 additions & 131 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,9 @@
11
// fallow-ignore-file complexity code-duplication
2-
/**
3-
* Audio Mixer Service
4-
*
5-
* Processes and mixes audio tracks using FFmpeg.
6-
*/
7-
82
import { isSelfOrAncestorHidden, memberGroupKey, isMemberGroupHidden } from "./mediaHidden.js";
93
import { closeSync, existsSync, mkdirSync, mkdtempSync, openSync, rmSync, writeFileSync } from "fs";
104
import { join, dirname, isAbsolute, relative } from "path";
115
import { parseHTML } from "linkedom";
12-
import { extractAudioMetadata } from "../utils/ffprobe.js";
6+
import { extractAudioMetadata, type AudioMetadata } from "../utils/ffprobe.js";
137
import { isNotMediaPayload } from "../utils/notMediaPayload.js";
148
import { clampAudioGain } from "@hyperframes/core/audio-gain";
159
import { clampFadesToDuration, readElementFades } from "@hyperframes/core/audio-fade";
@@ -61,19 +55,8 @@ import type { AudioVolumeKeyframe } from "./audioMixer.types.js";
6155

6256
export type { AudioElement, MixResult } from "./audioMixer.types.js";
6357

64-
/**
65-
* Filename every caller must use for the mixed-audio artifact.
66-
*
67-
* The extension is load-bearing, not cosmetic: FFmpeg picks the muxer from it,
68-
* and the mix is AAC-encoded. A raw ADTS `.aac` stream has nowhere to record
69-
* the encoder's priming delay, so those leading samples decode as real silence
70-
* and shift the whole track ~1024 samples (21.33 ms at 48 kHz) late against a
71-
* frame-accurate video track. An MP4-family container carries the delay as an
72-
* edit list, which every decoder then strips, so the mix lands on its authored
73-
* start. Keep the choice here rather than at each call site: the same file is
74-
* muxed into the video, shipped in a distributed plan, and handed to users as
75-
* the PNG-sequence sidecar, and all three have to agree.
76-
*/
58+
/** An MP4-family container records AAC priming as an edit list; a raw .aac plays ~21 ms late.
59+
* The video mux, the distributed plan and the PNG-sequence sidecar all read this one name. */
7760
export const MIXED_AUDIO_FILENAME = "audio.m4a";
7861

7962
function clampVolume(volume: number): number {
@@ -129,12 +112,13 @@ const MAX_RAMP_SLICES = 240;
129112
function buildRampFilterComplex(
130113
lane: RateSpec & object,
131114
duration: number,
115+
headFilter: string | null,
132116
tailFilter: string | null,
133117
): string {
134118
const slices = Math.min(MAX_RAMP_SLICES, Math.max(1, Math.ceil(duration / RAMP_SLICE_SECONDS)));
135119
const step = duration / slices;
136120
const split = Array.from({ length: slices }, (_, i) => `[s${i}]`).join("");
137-
const parts = [`[0:a]asplit=${slices}${split}`];
121+
const parts = [`[0:a]${headFilter ? `${headFilter},` : ""}asplit=${slices}${split}`];
138122
for (let i = 0; i < slices; i += 1) {
139123
const from = sourceTimeAt(lane, i * step);
140124
const to = sourceTimeAt(lane, (i + 1) * step);
@@ -156,27 +140,20 @@ function buildRampFilterComplex(
156140
}
157141

158142
function preparedAudioOutputArgs(
159-
srcPath: string,
143+
metadata: AudioMetadata | null,
160144
playbackRate: RateSpec,
161-
duration = 0,
162-
): Promise<string[]> {
163-
return stereoOutputArgs(srcPath).then((channelArgs) => {
164-
const filters: string[] = [];
165-
const outputArgs: string[] = [];
166-
if (channelArgs[0] === "-af" && channelArgs[1]) {
167-
filters.push(channelArgs[1]);
168-
} else {
169-
outputArgs.push(...channelArgs);
170-
}
171-
if (typeof playbackRate === "object") {
172-
const graph = buildRampFilterComplex(playbackRate, duration, filters.join(",") || null);
173-
return ["-filter_complex", graph, "-map", "[out]", ...outputArgs];
174-
}
175-
const atempo = buildAtempoFilter(playbackRate);
176-
if (atempo) filters.push(atempo);
177-
if (filters.length > 0) outputArgs.push("-af", filters.join(","));
178-
return outputArgs;
179-
});
145+
duration: number,
146+
headFilter: string | null,
147+
): string[] {
148+
const channelFilter = metadata?.channels === 1 ? STEREO_CHANNEL_FILTER : null;
149+
const outputArgs = channelFilter ? [] : ["-ac", "2"];
150+
if (typeof playbackRate === "object") {
151+
const graph = buildRampFilterComplex(playbackRate, duration, headFilter, channelFilter);
152+
return ["-filter_complex", graph, "-map", "[out]", ...outputArgs];
153+
}
154+
const filters = [headFilter, channelFilter, buildAtempoFilter(playbackRate)].filter(Boolean);
155+
if (filters.length > 0) outputArgs.push("-af", filters.join(","));
156+
return outputArgs;
180157
}
181158

182159
function escapeExpressionCommas(expression: string): string {
@@ -217,16 +194,6 @@ const VOLUME_SIMPLIFY_EPSILON = 0.005;
217194
// native stereo sources have FL/FR and pass through unchanged.
218195
const STEREO_CHANNEL_FILTER = "pan=stereo|FL=FL+FC|FR=FR+FC";
219196

220-
async function stereoOutputArgs(srcPath: string): Promise<string[]> {
221-
try {
222-
const { channels } = await extractAudioMetadata(srcPath);
223-
if (channels === 1) return ["-af", STEREO_CHANNEL_FILTER];
224-
} catch {
225-
// Preserve the previous FFmpeg conversion path when metadata probing fails.
226-
}
227-
return ["-ac", "2"];
228-
}
229-
230197
/**
231198
* Reduce a sorted keyframe list to a perceptually-equivalent piecewise-linear
232199
* envelope with a bounded segment count.
@@ -654,65 +621,12 @@ export function parseAudioElements(html: string): AudioElement[] {
654621
return elements;
655622
}
656623

657-
async function extractAudioFromVideo(
658-
videoPath: string,
659-
outputPath: string,
660-
options?: { startTime?: number; duration?: number; playbackRate?: RateSpec },
661-
signal?: AbortSignal,
662-
config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
663-
): Promise<ExtractResult> {
664-
const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
665-
const outputDir = dirname(outputPath);
666-
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
667-
668-
const playbackRate = normalizeRateSpec(options?.playbackRate);
669-
const args: string[] = [];
670-
if (options?.startTime !== undefined) args.push("-ss", String(options.startTime));
671-
if (options?.duration !== undefined) {
672-
args.push("-t", String(sourceTimeAt(playbackRate, options.duration)));
673-
}
674-
args.push("-i", videoPath);
675-
const outputArgs = await preparedAudioOutputArgs(videoPath, playbackRate, options?.duration);
676-
args.push("-vn", "-acodec", "pcm_s16le", "-ar", "48000", ...outputArgs);
677-
if (playbackRate !== 1 && options?.duration !== undefined) {
678-
args.push("-t", String(options.duration));
679-
}
680-
args.push("-y", outputPath);
681-
682-
const result = await runFfmpeg(args, { signal, timeout: ffmpegProcessTimeout });
683-
684-
if (signal?.aborted) {
685-
const failure: AudioProcessingFailure = {
686-
stage: "cancelled",
687-
reason: "cancelled",
688-
owner: "user",
689-
retryable: false,
690-
detail: "Audio extract cancelled",
691-
};
692-
return {
693-
success: false,
694-
outputPath,
695-
durationMs: result.durationMs,
696-
error: failure.detail,
697-
failure,
698-
};
699-
}
700-
if (!result.success) {
701-
const failure = ffmpegFailure("extract", result);
702-
return {
703-
success: false,
704-
outputPath,
705-
durationMs: result.durationMs,
706-
error: failure.detail,
707-
failure,
708-
};
709-
}
710-
return { success: true, outputPath, durationMs: result.durationMs };
711-
}
712-
713-
async function prepareAudioTrack(
624+
/** Decode `duration` s of audio from `mediaStart` in Chrome's media time. Until every stream has
625+
* started, an input seek snaps audio to the video's first keyframe, so read by timestamp instead. */
626+
async function extractAudioSegment(
714627
srcPath: string,
715628
outputPath: string,
629+
stage: "extract" | "prepare",
716630
mediaStart: number,
717631
duration: number,
718632
playbackRate: RateSpec = 1,
@@ -723,21 +637,25 @@ async function prepareAudioTrack(
723637
const outputDir = dirname(outputPath);
724638
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
725639
const normalizedPlaybackRate = normalizeRateSpec(playbackRate);
726-
const outputArgs = await preparedAudioOutputArgs(srcPath, normalizedPlaybackRate, duration);
640+
const sourceDuration = sourceTimeAt(normalizedPlaybackRate, duration);
641+
// A failed probe keeps FFmpeg's default channel conversion and the input seek.
642+
const metadata = await extractAudioMetadata(srcPath).catch(() => null);
643+
const inputSeek = mediaStart >= (metadata?.latestStreamLeadSeconds ?? 0);
644+
const start = formatFilterNumber(mediaStart);
645+
const timestampTrim = inputSeek
646+
? null
647+
: `atrim=start=${start}:end=${formatFilterNumber(mediaStart + sourceDuration)},asetpts=PTS-${start}/TB,aresample=async=1:first_pts=0`;
648+
const outputArgs = preparedAudioOutputArgs(
649+
metadata,
650+
normalizedPlaybackRate,
651+
duration,
652+
timestampTrim,
653+
);
727654

728-
const args = [
729-
"-ss",
730-
String(mediaStart),
731-
"-t",
732-
String(sourceTimeAt(normalizedPlaybackRate, duration)),
733-
"-i",
734-
srcPath,
735-
"-acodec",
736-
"pcm_s16le",
737-
"-ar",
738-
"48000",
739-
...outputArgs,
740-
];
655+
const args = inputSeek
656+
? ["-ss", String(mediaStart), "-t", String(sourceDuration), "-i", srcPath]
657+
: ["-i", srcPath];
658+
args.push("-vn", "-acodec", "pcm_s16le", "-ar", "48000", ...outputArgs);
741659
if (normalizedPlaybackRate !== 1) args.push("-t", String(duration));
742660
args.push("-y", outputPath);
743661

@@ -749,7 +667,7 @@ async function prepareAudioTrack(
749667
reason: "cancelled",
750668
owner: "user",
751669
retryable: false,
752-
detail: "Audio prepare cancelled",
670+
detail: `Audio ${stage} cancelled`,
753671
};
754672
return {
755673
success: false,
@@ -759,7 +677,7 @@ async function prepareAudioTrack(
759677
failure,
760678
};
761679
}
762-
const failure = !result.success ? ffmpegFailure("prepare", result) : undefined;
680+
const failure = !result.success ? ffmpegFailure(stage, result) : undefined;
763681
return {
764682
success: result.success,
765683
outputPath,
@@ -1243,14 +1161,13 @@ export async function processCompositionAudio(
12431161
let audioSrcPath = srcPath;
12441162
if (element.type === "video") {
12451163
const extractedPath = join(workDir, `${element.id}-extracted.wav`);
1246-
const extractResult = await extractAudioFromVideo(
1164+
const extractResult = await extractAudioSegment(
12471165
srcPath,
12481166
extractedPath,
1249-
{
1250-
startTime: element.mediaStart,
1251-
duration: element.end - element.start,
1252-
playbackRate: element.playbackRate,
1253-
},
1167+
"extract",
1168+
element.mediaStart,
1169+
element.end - element.start,
1170+
element.playbackRate,
12541171
effectiveSignal,
12551172
config,
12561173
);
@@ -1272,9 +1189,10 @@ export async function processCompositionAudio(
12721189
audioSrcPath = extractedPath;
12731190
} else {
12741191
const trimmedPath = join(workDir, `${element.id}-trimmed.wav`);
1275-
const prepResult = await prepareAudioTrack(
1192+
const prepResult = await extractAudioSegment(
12761193
srcPath,
12771194
trimmedPath,
1195+
"prepare",
12781196
element.mediaStart,
12791197
element.end - element.start,
12801198
element.playbackRate,

‎packages/engine/src/services/audioVolumeEnvelope.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
* output (and the golden baselines) only change where a fade is actually applied.
1212
*
1313
* The prepared tracks are always `pcm_s16le`, 48 kHz, stereo (see
14-
* `prepareAudioTrack` / `extractAudioFromVideo`). Anything else is rejected so
14+
* `extractAudioSegment`). Anything else is rejected so
1515
* the caller can fall back to the expression path rather than corrupting audio.
1616
*/
1717

0 commit comments

Comments
 (0)