Skip to content

Commit 2a7659e

Browse files
committed
fix(engine): audio from late-starting MP4 and WebM plays when the preview plays it
The mixer seeked and trimmed audio by FFmpeg's clock, which starts at the file's earliest stream. In an MP4 or WebM whose streams all start after 0, the browser plays that audio later, so the render heard it early by the same amount. The mixer now maps media time onto FFmpeg's clock with the same offset frame extraction uses.
1 parent 99a5bfb commit 2a7659e

4 files changed

Lines changed: 27 additions & 7 deletions

File tree

‎packages/engine/src/services/audioMixer.order.test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ vi.mock("./aacEncode.js", async (importOriginal) => ({
2323
encodeAac: vi.fn(async () => ({ success: true, durationMs: 1, stderr: "", exitCode: 0 })),
2424
}));
2525

26-
vi.mock("../utils/ffprobe.js", () => ({
26+
vi.mock("../utils/ffprobe.js", async (importOriginal) => ({
27+
...(await importOriginal<typeof import("../utils/ffprobe.js")>()),
2728
extractAudioMetadata: vi.fn(async () => ({ durationSeconds: 1, channels: 2, audioStream: 0 })),
2829
}));
2930

‎packages/engine/src/services/audioMixer.ts‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { renameWithRetry } from "@hyperframes/core/rename-retry";
55
import { removeScratch } from "@hyperframes/core/remove-scratch";
66
import { join, dirname, isAbsolute, relative } from "path";
77
import { parseHTML } from "linkedom";
8-
import { extractAudioMetadata, type AudioMetadata } from "../utils/ffprobe.js";
8+
import { extractAudioMetadata, ffmpegSeekSeconds, type AudioMetadata } from "../utils/ffprobe.js";
99
import { isNotMediaPayload } from "../utils/notMediaPayload.js";
1010
import { urlInputArgs } from "../utils/urlInput.js";
1111
import { clampAudioGain, clipGain } from "@hyperframes/core/audio-gain";
@@ -769,13 +769,14 @@ async function extractAudioSegment(
769769
removeScratch(cycleDir);
770770
}
771771
}
772-
const inputSeek = mediaStart >= (metadata?.latestStreamLeadSeconds ?? 0);
773-
const start = formatFilterNumber(mediaStart);
772+
const ffmpegStart = ffmpegSeekSeconds(metadata ?? {}, mediaStart);
773+
const inputSeek = ffmpegStart >= (metadata?.latestStreamLeadSeconds ?? 0);
774+
const start = formatFilterNumber(ffmpegStart);
774775
const timestampTrim = inputSeek
775776
? null
776-
: `atrim=start=${start}:end=${formatFilterNumber(mediaStart + sourceDuration)},asetpts=PTS-${start}/TB,aresample=async=1:first_pts=0`;
777+
: `atrim=start=${start}:end=${formatFilterNumber(ffmpegStart + sourceDuration)},asetpts=PTS-${start}/TB,aresample=async=1:first_pts=0`;
777778
const fromSidecar = audioSidecarUrl !== undefined && inputSeek && normalizedPlaybackRate === 1;
778-
const warmUp = fromSidecar ? Math.min(SIDECAR_WARM_UP_SECONDS, mediaStart) : 0;
779+
const warmUp = fromSidecar ? Math.min(SIDECAR_WARM_UP_SECONDS, ffmpegStart) : 0;
779780
const outputArgs = preparedAudioOutputArgs(
780781
metadata,
781782
normalizedPlaybackRate,
@@ -794,7 +795,7 @@ async function extractAudioSegment(
794795
...(readsUrl ? FAIL_ON_READ_ERROR_INSTEAD_OF_ENDING_THERE : []),
795796
...urlInputArgs(readFrom),
796797
...(inputSeek
797-
? ["-ss", String(mediaStart - warmUp), "-t", String(sourceDuration + warmUp)]
798+
? ["-ss", String(ffmpegStart - warmUp), "-t", String(sourceDuration + warmUp)]
798799
: []),
799800
"-i",
800801
readFrom,

‎packages/engine/src/services/streamStartOffset.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ const SAMPLE_RATE = 48_000;
1919
const LEAD_SECONDS = 7;
2020
const CLIP_SECONDS = 12;
2121
const PICTURE_SECONDS = 20;
22+
const BOTH_LATE_SECONDS = 2;
2223

2324
function ffmpeg(args: string[]): void {
2425
const result = spawnSync(FFMPEG, ["-nostdin", "-v", "error", "-y", ...args], {
@@ -89,6 +90,17 @@ describe.skipIf(!HAS_FFMPEG)(
8990
const mux = ["-map", "0:v", "-map", "1:a", "-c", "copy"];
9091
ffmpeg(["-itsoffset", lead, "-i", picture, "-i", steps, ...mux, join(dir, "video-late.mp4")]);
9192
ffmpeg(["-i", picture, "-itsoffset", lead, "-i", steps, ...mux, join(dir, "audio-late.mp4")]);
93+
const both = String(BOTH_LATE_SECONDS);
94+
ffmpeg([
95+
"-i",
96+
picture,
97+
"-i",
98+
steps,
99+
...mux,
100+
"-output_ts_offset",
101+
both,
102+
join(dir, "both-late.mp4"),
103+
]);
92104
// MPEG-TS starts its clock at 1.4 s, so no stream starts at 0.
93105
ffmpeg(["-i", join(dir, "video-late.mp4"), "-c", "copy", join(dir, "video-late.ts")]);
94106
}, 60_000);
@@ -111,6 +123,9 @@ describe.skipIf(!HAS_FFMPEG)(
111123
{ source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "video", mediaStart: 0 },
112124
{ source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "video", mediaStart: 3 },
113125
{ source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "audio", mediaStart: 0 },
126+
{ source: "both-late.mp4", audioLead: BOTH_LATE_SECONDS, type: "audio", mediaStart: 0 },
127+
{ source: "both-late.mp4", audioLead: BOTH_LATE_SECONDS, type: "video", mediaStart: 1 },
128+
{ source: "both-late.mp4", audioLead: BOTH_LATE_SECONDS, type: "audio", mediaStart: 3 },
114129
{
115130
source: "audio-late.mp4",
116131
audioLead: LEAD_SECONDS,

‎packages/engine/src/utils/ffprobe.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,8 @@ export interface AudioMetadata {
245245
/** Seconds after the file's media time 0 at which its last audio or video stream starts. */
246246
latestStreamLeadSeconds?: number;
247247
audioStreamLeadSeconds?: number;
248+
/** As on VideoMetadata: how far the browser's media time runs ahead of the file's. */
249+
mediaTimeOffsetSeconds?: number;
248250
audioStreamCount?: number;
249251
/** The track read for these facts and for the mix, by its place among the audio streams (decodableAudioStream). */
250252
audioStream: number;
@@ -1267,6 +1269,7 @@ async function probeAudioMetadata(filePath: string, signal?: AbortSignal): Promi
12671269
bitrate: output.format.bit_rate ? parseInt(output.format.bit_rate) : undefined,
12681270
latestStreamLeadSeconds: latestStreamLeadSeconds(output),
12691271
audioStreamLeadSeconds: streamLeadSeconds(output, audioStream),
1272+
mediaTimeOffsetSeconds: mediaTimeOffsetSeconds(output),
12701273
audioStreamCount: audioStreams.length,
12711274
audioStream: audioStreamIndex,
12721275
};

0 commit comments

Comments
 (0)