Skip to content

Commit 63b68f7

Browse files
committed
fix(engine): a late-starting file's picture shows on the clock its audio plays on
1 parent 2a7659e commit 63b68f7

3 files changed

Lines changed: 26 additions & 9 deletions

File tree

‎packages/engine/src/services/streamStartOffset.test.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,26 @@ describe.skipIf(!HAS_FFMPEG)(
183183
expect(getFrameAtTime(extracted, 8, 0)).not.toBeNull();
184184
}, 30_000);
185185

186+
it("shows the picture of a file whose streams all start late on the clock its audio plays on", async () => {
187+
const frames = join(dir, "frames");
188+
const source = await extractVideoFramesRange(join(dir, "picture.mp4"), "picture", 0, 5, {
189+
fps: 30,
190+
outputDir: frames,
191+
});
192+
const late = await extractVideoFramesRange(
193+
join(dir, "both-late.mp4"),
194+
"both-late",
195+
0,
196+
5 + BOTH_LATE_SECONDS,
197+
{ fps: 30, outputDir: frames },
198+
);
199+
for (const second of [0.5, 3.5]) {
200+
expect(readFileSync(getFrameAtTime(late, BOTH_LATE_SECONDS + second, 0)!)).toEqual(
201+
readFileSync(getFrameAtTime(source, second, 0)!),
202+
);
203+
}
204+
}, 30_000);
205+
186206
it("keeps every frame of a video stream that starts first", async () => {
187207
const extracted = await extractVideoFramesRange(
188208
join(dir, "audio-late.mp4"),

‎packages/engine/src/services/videoFrameExtractor.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1621,9 +1621,9 @@ export async function resolveFinalFrameExtractionWindow(
16211621
if (!window.ensureFinalFrame) return window;
16221622
const playableDuration = resolvePlayableVideoDuration(metadata);
16231623
const videoStreamLead = metadata.videoStreamLeadSeconds ?? 0;
1624-
// A file can run past the duration it reports; the browser then holds the frame at that end.
1624+
const lastInstantTheBrowserPlays = playableDuration - FINAL_FRAME_LOGICAL_DURATION_SECONDS;
16251625
const finalFrameTimestamp = Math.min(
1626-
playableDuration - FINAL_FRAME_LOGICAL_DURATION_SECONDS,
1626+
lastInstantTheBrowserPlays,
16271627
videoStartOnMediaTimeline(metadata) +
16281628
(await extractFinalVideoFrameTimestamp(
16291629
videoPath,

‎packages/engine/src/utils/ffprobe.ts‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,6 @@ export interface VideoMetadata {
207207
videoStreamStartSeconds?: number;
208208
/** Seconds after the file's media time 0 (its earliest audio/video stream start) at which video begins. */
209209
videoStreamLeadSeconds?: number;
210-
/** How far the browser's media time runs ahead of the file's: it counts an MP4 or WebM from timestamp 0. */
211210
mediaTimeOffsetSeconds?: number;
212211
width: number;
213212
height: number;
@@ -245,7 +244,6 @@ export interface AudioMetadata {
245244
/** Seconds after the file's media time 0 at which its last audio or video stream starts. */
246245
latestStreamLeadSeconds?: number;
247246
audioStreamLeadSeconds?: number;
248-
/** As on VideoMetadata: how far the browser's media time runs ahead of the file's. */
249247
mediaTimeOffsetSeconds?: number;
250248
audioStreamCount?: number;
251249
/** The track read for these facts and for the mix, by its place among the audio streams (decodableAudioStream). */
@@ -516,14 +514,12 @@ export async function buildProbeFactsText(filePath: string, signal?: AbortSignal
516514
return JSON.stringify(facts);
517515
}
518516

519-
/** Where the video begins on the media timeline the browser plays. */
520517
export function videoStartOnMediaTimeline(
521518
metadata: Pick<VideoMetadata, "videoStreamLeadSeconds" | "mediaTimeOffsetSeconds">,
522519
): number {
523520
return (metadata.videoStreamLeadSeconds ?? 0) + (metadata.mediaTimeOffsetSeconds ?? 0);
524521
}
525522

526-
/** FFmpeg's input `-ss` for a media time: FFmpeg counts from the file's earliest stream start. */
527523
export function ffmpegSeekSeconds(
528524
metadata: Pick<VideoMetadata, "mediaTimeOffsetSeconds">,
529525
mediaTime: number,
@@ -1292,11 +1288,12 @@ function streamLeadSeconds(output: FFProbeOutput | null, stream: FFProbeStream):
12921288
return Number.isFinite(start) && starts.length > 0 ? Math.max(0, start - Math.min(...starts)) : 0;
12931289
}
12941290

1295-
// Transport streams start their clock anywhere (MPEG-TS at 1.4 s); their earliest stream is media time 0.
1291+
const BROWSER_PLAYS_FROM_TIMESTAMP_ZERO = /\b(mp4|webm)\b/;
1292+
12961293
function mediaTimeOffsetSeconds(output: FFProbeOutput | null): number {
12971294
const starts = audioVideoStreamStarts(output);
1298-
const countsFromZero = /\b(mp4|webm)\b/.test(output?.format.format_name ?? "");
1299-
return countsFromZero && starts.length > 0 ? Math.max(0, Math.min(...starts)) : 0;
1295+
const fromZero = BROWSER_PLAYS_FROM_TIMESTAMP_ZERO.test(output?.format.format_name ?? "");
1296+
return fromZero && starts.length > 0 ? Math.max(0, Math.min(...starts)) : 0;
13001297
}
13011298

13021299
function latestStreamLeadSeconds(output: FFProbeOutput): number {

0 commit comments

Comments
 (0)