11// fallow-ignore-file complexity code-duplication
2- /**
3- * Audio Mixer Service
4- *
5- * Processes and mixes audio tracks using FFmpeg.
6- */
7-
82import { isSelfOrAncestorHidden , memberGroupKey , isMemberGroupHidden } from "./mediaHidden.js" ;
93import { closeSync , existsSync , mkdirSync , mkdtempSync , openSync , rmSync , writeFileSync } from "fs" ;
104import { join , dirname , isAbsolute , relative } from "path" ;
115import { parseHTML } from "linkedom" ;
12- import { extractAudioMetadata } from "../utils/ffprobe.js" ;
6+ import { extractAudioMetadata , type AudioMetadata } from "../utils/ffprobe.js" ;
137import { isNotMediaPayload } from "../utils/notMediaPayload.js" ;
148import { clampAudioGain } from "@hyperframes/core/audio-gain" ;
159import { clampFadesToDuration , readElementFades } from "@hyperframes/core/audio-fade" ;
@@ -61,19 +55,8 @@ import type { AudioVolumeKeyframe } from "./audioMixer.types.js";
6155
6256export type { AudioElement , MixResult } from "./audioMixer.types.js" ;
6357
64- /**
65- * Filename every caller must use for the mixed-audio artifact.
66- *
67- * The extension is load-bearing, not cosmetic: FFmpeg picks the muxer from it,
68- * and the mix is AAC-encoded. A raw ADTS `.aac` stream has nowhere to record
69- * the encoder's priming delay, so those leading samples decode as real silence
70- * and shift the whole track ~1024 samples (21.33 ms at 48 kHz) late against a
71- * frame-accurate video track. An MP4-family container carries the delay as an
72- * edit list, which every decoder then strips, so the mix lands on its authored
73- * start. Keep the choice here rather than at each call site: the same file is
74- * muxed into the video, shipped in a distributed plan, and handed to users as
75- * the PNG-sequence sidecar, and all three have to agree.
76- */
58+ /** An MP4-family container records AAC priming as an edit list; a raw .aac plays ~21 ms late.
59+ * The video mux, the distributed plan and the PNG-sequence sidecar all read this one name. */
7760export const MIXED_AUDIO_FILENAME = "audio.m4a" ;
7861
7962function clampVolume ( volume : number ) : number {
@@ -129,12 +112,13 @@ const MAX_RAMP_SLICES = 240;
129112function buildRampFilterComplex (
130113 lane : RateSpec & object ,
131114 duration : number ,
115+ headFilter : string | null ,
132116 tailFilter : string | null ,
133117) : string {
134118 const slices = Math . min ( MAX_RAMP_SLICES , Math . max ( 1 , Math . ceil ( duration / RAMP_SLICE_SECONDS ) ) ) ;
135119 const step = duration / slices ;
136120 const split = Array . from ( { length : slices } , ( _ , i ) => `[s${ i } ]` ) . join ( "" ) ;
137- const parts = [ `[0:a]asplit=${ slices } ${ split } ` ] ;
121+ const parts = [ `[0:a]${ headFilter ? ` ${ headFilter } ,` : "" } asplit=${ slices } ${ split } ` ] ;
138122 for ( let i = 0 ; i < slices ; i += 1 ) {
139123 const from = sourceTimeAt ( lane , i * step ) ;
140124 const to = sourceTimeAt ( lane , ( i + 1 ) * step ) ;
@@ -156,27 +140,20 @@ function buildRampFilterComplex(
156140}
157141
158142function preparedAudioOutputArgs (
159- srcPath : string ,
143+ metadata : AudioMetadata | null ,
160144 playbackRate : RateSpec ,
161- duration = 0 ,
162- ) : Promise < string [ ] > {
163- return stereoOutputArgs ( srcPath ) . then ( ( channelArgs ) => {
164- const filters : string [ ] = [ ] ;
165- const outputArgs : string [ ] = [ ] ;
166- if ( channelArgs [ 0 ] === "-af" && channelArgs [ 1 ] ) {
167- filters . push ( channelArgs [ 1 ] ) ;
168- } else {
169- outputArgs . push ( ...channelArgs ) ;
170- }
171- if ( typeof playbackRate === "object" ) {
172- const graph = buildRampFilterComplex ( playbackRate , duration , filters . join ( "," ) || null ) ;
173- return [ "-filter_complex" , graph , "-map" , "[out]" , ...outputArgs ] ;
174- }
175- const atempo = buildAtempoFilter ( playbackRate ) ;
176- if ( atempo ) filters . push ( atempo ) ;
177- if ( filters . length > 0 ) outputArgs . push ( "-af" , filters . join ( "," ) ) ;
178- return outputArgs ;
179- } ) ;
145+ duration : number ,
146+ headFilter : string | null ,
147+ ) : string [ ] {
148+ const channelFilter = metadata ?. channels === 1 ? STEREO_CHANNEL_FILTER : null ;
149+ const outputArgs = channelFilter ? [ ] : [ "-ac" , "2" ] ;
150+ if ( typeof playbackRate === "object" ) {
151+ const graph = buildRampFilterComplex ( playbackRate , duration , headFilter , channelFilter ) ;
152+ return [ "-filter_complex" , graph , "-map" , "[out]" , ...outputArgs ] ;
153+ }
154+ const filters = [ headFilter , channelFilter , buildAtempoFilter ( playbackRate ) ] . filter ( Boolean ) ;
155+ if ( filters . length > 0 ) outputArgs . push ( "-af" , filters . join ( "," ) ) ;
156+ return outputArgs ;
180157}
181158
182159function escapeExpressionCommas ( expression : string ) : string {
@@ -217,16 +194,6 @@ const VOLUME_SIMPLIFY_EPSILON = 0.005;
217194// native stereo sources have FL/FR and pass through unchanged.
218195const STEREO_CHANNEL_FILTER = "pan=stereo|FL=FL+FC|FR=FR+FC" ;
219196
220- async function stereoOutputArgs ( srcPath : string ) : Promise < string [ ] > {
221- try {
222- const { channels } = await extractAudioMetadata ( srcPath ) ;
223- if ( channels === 1 ) return [ "-af" , STEREO_CHANNEL_FILTER ] ;
224- } catch {
225- // Preserve the previous FFmpeg conversion path when metadata probing fails.
226- }
227- return [ "-ac" , "2" ] ;
228- }
229-
230197/**
231198 * Reduce a sorted keyframe list to a perceptually-equivalent piecewise-linear
232199 * envelope with a bounded segment count.
@@ -654,65 +621,12 @@ export function parseAudioElements(html: string): AudioElement[] {
654621 return elements ;
655622}
656623
657- async function extractAudioFromVideo (
658- videoPath : string ,
659- outputPath : string ,
660- options ?: { startTime ?: number ; duration ?: number ; playbackRate ?: RateSpec } ,
661- signal ?: AbortSignal ,
662- config ?: Partial < Pick < EngineConfig , "ffmpegProcessTimeout" > > ,
663- ) : Promise < ExtractResult > {
664- const ffmpegProcessTimeout = config ?. ffmpegProcessTimeout ?? DEFAULT_CONFIG . ffmpegProcessTimeout ;
665- const outputDir = dirname ( outputPath ) ;
666- if ( ! existsSync ( outputDir ) ) mkdirSync ( outputDir , { recursive : true } ) ;
667-
668- const playbackRate = normalizeRateSpec ( options ?. playbackRate ) ;
669- const args : string [ ] = [ ] ;
670- if ( options ?. startTime !== undefined ) args . push ( "-ss" , String ( options . startTime ) ) ;
671- if ( options ?. duration !== undefined ) {
672- args . push ( "-t" , String ( sourceTimeAt ( playbackRate , options . duration ) ) ) ;
673- }
674- args . push ( "-i" , videoPath ) ;
675- const outputArgs = await preparedAudioOutputArgs ( videoPath , playbackRate , options ?. duration ) ;
676- args . push ( "-vn" , "-acodec" , "pcm_s16le" , "-ar" , "48000" , ...outputArgs ) ;
677- if ( playbackRate !== 1 && options ?. duration !== undefined ) {
678- args . push ( "-t" , String ( options . duration ) ) ;
679- }
680- args . push ( "-y" , outputPath ) ;
681-
682- const result = await runFfmpeg ( args , { signal, timeout : ffmpegProcessTimeout } ) ;
683-
684- if ( signal ?. aborted ) {
685- const failure : AudioProcessingFailure = {
686- stage : "cancelled" ,
687- reason : "cancelled" ,
688- owner : "user" ,
689- retryable : false ,
690- detail : "Audio extract cancelled" ,
691- } ;
692- return {
693- success : false ,
694- outputPath,
695- durationMs : result . durationMs ,
696- error : failure . detail ,
697- failure,
698- } ;
699- }
700- if ( ! result . success ) {
701- const failure = ffmpegFailure ( "extract" , result ) ;
702- return {
703- success : false ,
704- outputPath,
705- durationMs : result . durationMs ,
706- error : failure . detail ,
707- failure,
708- } ;
709- }
710- return { success : true , outputPath, durationMs : result . durationMs } ;
711- }
712-
713- async function prepareAudioTrack (
624+ /** Decode `duration` s of audio from `mediaStart` in Chrome's media time. Until every stream has
625+ * started, an input seek snaps audio to the video's first keyframe, so read by timestamp instead. */
626+ async function extractAudioSegment (
714627 srcPath : string ,
715628 outputPath : string ,
629+ stage : "extract" | "prepare" ,
716630 mediaStart : number ,
717631 duration : number ,
718632 playbackRate : RateSpec = 1 ,
@@ -723,21 +637,25 @@ async function prepareAudioTrack(
723637 const outputDir = dirname ( outputPath ) ;
724638 if ( ! existsSync ( outputDir ) ) mkdirSync ( outputDir , { recursive : true } ) ;
725639 const normalizedPlaybackRate = normalizeRateSpec ( playbackRate ) ;
726- const outputArgs = await preparedAudioOutputArgs ( srcPath , normalizedPlaybackRate , duration ) ;
640+ const sourceDuration = sourceTimeAt ( normalizedPlaybackRate , duration ) ;
641+ // A failed probe keeps FFmpeg's default channel conversion and the input seek.
642+ const metadata = await extractAudioMetadata ( srcPath ) . catch ( ( ) => null ) ;
643+ const inputSeek = mediaStart >= ( metadata ?. latestStreamLeadSeconds ?? 0 ) ;
644+ const start = formatFilterNumber ( mediaStart ) ;
645+ const timestampTrim = inputSeek
646+ ? null
647+ : `atrim=start=${ start } :end=${ formatFilterNumber ( mediaStart + sourceDuration ) } ,asetpts=PTS-${ start } /TB,aresample=async=1:first_pts=0` ;
648+ const outputArgs = preparedAudioOutputArgs (
649+ metadata ,
650+ normalizedPlaybackRate ,
651+ duration ,
652+ timestampTrim ,
653+ ) ;
727654
728- const args = [
729- "-ss" ,
730- String ( mediaStart ) ,
731- "-t" ,
732- String ( sourceTimeAt ( normalizedPlaybackRate , duration ) ) ,
733- "-i" ,
734- srcPath ,
735- "-acodec" ,
736- "pcm_s16le" ,
737- "-ar" ,
738- "48000" ,
739- ...outputArgs ,
740- ] ;
655+ const args = inputSeek
656+ ? [ "-ss" , String ( mediaStart ) , "-t" , String ( sourceDuration ) , "-i" , srcPath ]
657+ : [ "-i" , srcPath ] ;
658+ args . push ( "-vn" , "-acodec" , "pcm_s16le" , "-ar" , "48000" , ...outputArgs ) ;
741659 if ( normalizedPlaybackRate !== 1 ) args . push ( "-t" , String ( duration ) ) ;
742660 args . push ( "-y" , outputPath ) ;
743661
@@ -749,7 +667,7 @@ async function prepareAudioTrack(
749667 reason : "cancelled" ,
750668 owner : "user" ,
751669 retryable : false ,
752- detail : " Audio prepare cancelled" ,
670+ detail : ` Audio ${ stage } cancelled` ,
753671 } ;
754672 return {
755673 success : false ,
@@ -759,7 +677,7 @@ async function prepareAudioTrack(
759677 failure,
760678 } ;
761679 }
762- const failure = ! result . success ? ffmpegFailure ( "prepare" , result ) : undefined ;
680+ const failure = ! result . success ? ffmpegFailure ( stage , result ) : undefined ;
763681 return {
764682 success : result . success ,
765683 outputPath,
@@ -1243,14 +1161,13 @@ export async function processCompositionAudio(
12431161 let audioSrcPath = srcPath ;
12441162 if ( element . type === "video" ) {
12451163 const extractedPath = join ( workDir , `${ element . id } -extracted.wav` ) ;
1246- const extractResult = await extractAudioFromVideo (
1164+ const extractResult = await extractAudioSegment (
12471165 srcPath ,
12481166 extractedPath ,
1249- {
1250- startTime : element . mediaStart ,
1251- duration : element . end - element . start ,
1252- playbackRate : element . playbackRate ,
1253- } ,
1167+ "extract" ,
1168+ element . mediaStart ,
1169+ element . end - element . start ,
1170+ element . playbackRate ,
12541171 effectiveSignal ,
12551172 config ,
12561173 ) ;
@@ -1272,9 +1189,10 @@ export async function processCompositionAudio(
12721189 audioSrcPath = extractedPath ;
12731190 } else {
12741191 const trimmedPath = join ( workDir , `${ element . id } -trimmed.wav` ) ;
1275- const prepResult = await prepareAudioTrack (
1192+ const prepResult = await extractAudioSegment (
12761193 srcPath ,
12771194 trimmedPath ,
1195+ "prepare" ,
12781196 element . mediaStart ,
12791197 element . end - element . start ,
12801198 element . playbackRate ,
0 commit comments