@@ -68,6 +68,9 @@ import { RATE_RANGE } from "@hyperframes/core/audio-automation";
6868// below run too — they exercise the extractor in isolation against a
6969// synthesized VFR fixture.
7070const HAS_FFMPEG = spawnSync ( "ffmpeg" , [ "-version" ] ) . status === 0 ;
71+ const HAS_ZSCALE =
72+ HAS_FFMPEG &&
73+ / \s z s c a l e \s / . test ( spawnSync ( "ffmpeg" , [ "-hide_banner" , "-filters" ] ) . stdout . toString ( ) ) ;
7174
7275describe ( "resolveVideoExtractionDuration" , ( ) => {
7376 const metadata = (
@@ -3140,7 +3143,17 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
31403143 } , 60_000 ) ;
31413144} ) ;
31423145
3143- describe . skipIf ( ! HAS_FFMPEG || process . platform === "darwin" ) ( "forced-SDR HDR extraction" , ( ) => {
3146+ // Release builds print "version 8.1.3" or "n8.1.3"; master builds print "N-<number>" and are newer.
3147+ const FFPROBE_VERSION = spawnSync ( "ffprobe" , [ "-version" ] ) . stdout ?. toString ( ) ?? "" ;
3148+ const FFPROBE_RELEASE = FFPROBE_VERSION . match ( / v e r s i o n n ? ( \d + ) \. ( \d + ) / )
3149+ ?. slice ( 1 )
3150+ . map ( Number ) ;
3151+ const FFPROBE_READS_CONTAINER_TRANSFER =
3152+ / v e r s i o n N - / . test ( FFPROBE_VERSION ) ||
3153+ ( FFPROBE_RELEASE !== undefined &&
3154+ ( FFPROBE_RELEASE [ 0 ] ! > 8 || ( FFPROBE_RELEASE [ 0 ] === 8 && FFPROBE_RELEASE [ 1 ] ! >= 1 ) ) ) ;
3155+
3156+ describe . skipIf ( ! HAS_ZSCALE ) ( "forced-SDR HDR extraction" , ( ) => {
31443157 let fixtureDir = "" ;
31453158
31463159 beforeAll ( ( ) => {
@@ -3151,8 +3164,11 @@ describe.skipIf(!HAS_FFMPEG || process.platform === "darwin")("forced-SDR HDR ex
31513164 rmSync ( fixtureDir , { recursive : true , force : true } ) ;
31523165 } ) ;
31533166
3154- it ( "matches Studio's HLG tone map and isolates transformed cache entries" , async ( ) => {
3155- const source = join ( fixtureDir , "hlg-warm.mp4" ) ;
3167+ async function synthesizeHlgClip (
3168+ path : string ,
3169+ vui = "colour_primaries=9:transfer_characteristics=18:matrix_coefficients=9" ,
3170+ transfer = "arib-std-b67" ,
3171+ ) : Promise < void > {
31563172 const synthesized = await runFfmpeg ( [
31573173 "-y" ,
31583174 "-hide_banner" ,
@@ -3163,24 +3179,205 @@ describe.skipIf(!HAS_FFMPEG || process.platform === "darwin")("forced-SDR HDR ex
31633179 "-i" ,
31643180 "color=c=0xe0b080:s=64x64:r=1:d=1" ,
31653181 "-vf" ,
3166- " zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=arib-std-b67 :m=bt2020nc:r=tv,format=yuv420p" ,
3182+ ` zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=${ transfer } :m=bt2020nc:r=tv,format=yuv420p` ,
31673183 "-c:v" ,
31683184 "libx264" ,
31693185 "-preset" ,
31703186 "ultrafast" ,
31713187 "-color_primaries" ,
31723188 "bt2020" ,
31733189 "-color_trc" ,
3174- "arib-std-b67" ,
3190+ transfer ,
31753191 "-colorspace" ,
31763192 "bt2020nc" ,
31773193 "-bsf:v" ,
3178- " h264_metadata=colour_primaries=9:transfer_characteristics=18:matrix_coefficients=9" ,
3179- source ,
3194+ ` h264_metadata=${ vui } ` ,
3195+ path ,
31803196 ] ) ;
31813197 if ( ! synthesized . success ) {
31823198 throw new Error ( `HLG fixture synthesis failed: ${ synthesized . stderr . slice ( - 400 ) } ` ) ;
31833199 }
3200+ }
3201+
3202+ it . each ( [
3203+ [
3204+ "HLG" ,
3205+ "matrix" ,
3206+ "arib-std-b67" ,
3207+ "colour_primaries=9:transfer_characteristics=18:matrix_coefficients=2" ,
3208+ ] ,
3209+ [
3210+ "HLG" ,
3211+ "primaries" ,
3212+ "arib-std-b67" ,
3213+ "colour_primaries=2:transfer_characteristics=18:matrix_coefficients=9" ,
3214+ ] ,
3215+ [
3216+ "HLG" ,
3217+ "matrix and primaries" ,
3218+ "arib-std-b67" ,
3219+ "colour_primaries=2:transfer_characteristics=18:matrix_coefficients=2" ,
3220+ ] ,
3221+ [
3222+ "PQ" ,
3223+ "matrix and primaries" ,
3224+ "smpte2084" ,
3225+ "colour_primaries=2:transfer_characteristics=16:matrix_coefficients=2" ,
3226+ ] ,
3227+ [
3228+ "HLG" ,
3229+ "transfer" ,
3230+ "arib-std-b67" ,
3231+ "colour_primaries=9:transfer_characteristics=2:matrix_coefficients=9" ,
3232+ ] ,
3233+ ] ) (
3234+ "tone-maps %s footage whose video stream has no %s tag like the fully tagged clip" ,
3235+ async ( name , missing , transfer , vui ) => {
3236+ const slug = `${ name } -no-${ missing } ` . replace ( / \W + / g, "-" ) ;
3237+ const tagged = join ( fixtureDir , `${ name } -fully-tagged.mp4` ) ;
3238+ const untagged = join ( fixtureDir , `${ slug } .mp4` ) ;
3239+ const vuiTransfer = transfer === "smpte2084" ? 16 : 18 ;
3240+ await synthesizeHlgClip (
3241+ tagged ,
3242+ `colour_primaries=9:transfer_characteristics=${ vuiTransfer } :matrix_coefficients=9` ,
3243+ transfer ,
3244+ ) ;
3245+ await synthesizeHlgClip ( untagged , vui , transfer ) ;
3246+ const extract = ( source : string , id : string ) =>
3247+ extractVideoFramesRange ( source , id , 0 , 1 , {
3248+ fps : 1 ,
3249+ outputDir : join ( fixtureDir , `out-${ id } ` ) ,
3250+ format : "png" ,
3251+ toneMapHdrToSdr : true ,
3252+ } ) ;
3253+
3254+ if ( missing === "transfer" ) {
3255+ // ffprobe before 8.1 reads the stream's unspecified transfer, so the clip is SDR there, as on main.
3256+ if ( ! FFPROBE_READS_CONTAINER_TRANSFER ) {
3257+ await expect ( extract ( untagged , slug ) ) . resolves . toBeDefined ( ) ;
3258+ return ;
3259+ }
3260+ expect ( ( await extractVideoMetadata ( untagged ) ) . colorSpace ?. colorTransfer ) . toBe ( transfer ) ;
3261+ }
3262+ const reference = await extract ( tagged , `ref-${ slug } ` ) ;
3263+ const result = await extract ( untagged , slug ) ;
3264+
3265+ expect ( readFileSync ( result . framePaths . get ( 0 ) ! ) ) . toEqual (
3266+ readFileSync ( reference . framePaths . get ( 0 ) ! ) ,
3267+ ) ;
3268+ } ,
3269+ 60_000 ,
3270+ ) ;
3271+
3272+ // One stream with HLG frames, then PQ frames: each is tone-mapped with its own transfer.
3273+ it ( "tone-maps each frame of a mixed HLG and PQ stream with its own transfer" , async ( ) => {
3274+ const segment = async ( name : string , transfer : number ) => {
3275+ const path = join ( fixtureDir , `mixed-${ name } .h264` ) ;
3276+ const result = await runFfmpeg ( [
3277+ "-y" ,
3278+ "-v" ,
3279+ "error" ,
3280+ "-f" ,
3281+ "lavfi" ,
3282+ "-i" ,
3283+ "testsrc2=s=160x90:r=25:d=0.2" ,
3284+ "-c:v" ,
3285+ "libx264" ,
3286+ // No B-frames: genpts stamps a raw stream in decode order, so reordered frames get wrong times.
3287+ "-bf" ,
3288+ "0" ,
3289+ "-pix_fmt" ,
3290+ "yuv420p" ,
3291+ "-bsf:v" ,
3292+ `h264_metadata=colour_primaries=9:transfer_characteristics=${ transfer } :matrix_coefficients=9` ,
3293+ "-f" ,
3294+ "h264" ,
3295+ path ,
3296+ ] ) ;
3297+ if ( ! result . success )
3298+ throw new Error ( `segment synthesis failed: ${ result . stderr . slice ( - 400 ) } ` ) ;
3299+ return readFileSync ( path ) ;
3300+ } ;
3301+ const mux = async ( name : string , stream : Buffer ) => {
3302+ const raw = join ( fixtureDir , `${ name } .h264` ) ;
3303+ const path = join ( fixtureDir , `${ name } .mp4` ) ;
3304+ writeFileSync ( raw , stream ) ;
3305+ const result = await runFfmpeg ( [
3306+ ...[ "-y" , "-v" , "error" , "-fflags" , "+genpts" , "-r" , "25" , "-f" , "h264" , "-i" , raw ] ,
3307+ ...[ "-c" , "copy" , path ] ,
3308+ ] ) ;
3309+ if ( ! result . success ) throw new Error ( `mux failed: ${ result . stderr . slice ( - 400 ) } ` ) ;
3310+ return path ;
3311+ } ;
3312+ const hlg = await segment ( "hlg" , 18 ) ;
3313+ const pq = await segment ( "pq" , 16 ) ;
3314+ const mixed = await mux ( "mixed-hlg-pq" , Buffer . concat ( [ hlg , pq ] ) ) ;
3315+ const pqOnly = await mux ( "mixed-pq-only" , pq ) ;
3316+ const extract = ( source : string , id : string , duration : number ) =>
3317+ extractVideoFramesRange ( source , id , 0 , duration , {
3318+ fps : 25 ,
3319+ outputDir : join ( fixtureDir , `out-${ id } ` ) ,
3320+ format : "png" ,
3321+ toneMapHdrToSdr : true ,
3322+ } ) ;
3323+
3324+ const mixedFrames = await extract ( mixed , "mixed-hlg-pq" , 0.4 ) ;
3325+ const pqFrames = await extract ( pqOnly , "mixed-pq-only" , 0.2 ) ;
3326+
3327+ expect ( readFileSync ( mixedFrames . framePaths . get ( 5 ) ! ) ) . toEqual (
3328+ readFileSync ( pqFrames . framePaths . get ( 0 ) ! ) ,
3329+ ) ;
3330+ } , 60_000 ) ;
3331+
3332+ // A stream-copy trim opens on edit-list pre-roll, which ffprobe counts as packets without frames.
3333+ it ( "keeps per-frame tags in a mixed HLG and PQ file trimmed with a stream copy" , async ( ) => {
3334+ const run = async ( args : string [ ] ) => {
3335+ const result = await runFfmpeg ( [ "-y" , "-v" , "error" , ...args ] ) ;
3336+ if ( ! result . success ) throw new Error ( `fixture failed: ${ result . stderr . slice ( - 400 ) } ` ) ;
3337+ } ;
3338+ const segment = async ( name : string , transfer : number ) => {
3339+ const path = join ( fixtureDir , `trim-${ name } .h264` ) ;
3340+ await run ( [
3341+ ...[ "-f" , "lavfi" , "-i" , "testsrc2=s=160x90:r=25:d=2" ] ,
3342+ ...[ "-c:v" , "libx264" , "-g" , "50" , "-bf" , "0" ] ,
3343+ ...[ "-pix_fmt" , "yuv420p" , "-bsf:v" ] ,
3344+ `h264_metadata=colour_primaries=9:transfer_characteristics=${ transfer } :matrix_coefficients=9` ,
3345+ ...[ "-f" , "h264" , path ] ,
3346+ ] ) ;
3347+ return readFileSync ( path ) ;
3348+ } ;
3349+ const mux = async ( name : string , stream : Buffer ) => {
3350+ const raw = join ( fixtureDir , `${ name } .h264` ) ;
3351+ const path = join ( fixtureDir , `${ name } .mp4` ) ;
3352+ writeFileSync ( raw , stream ) ;
3353+ await run ( [ "-fflags" , "+genpts" , "-r" , "25" , "-f" , "h264" , "-i" , raw , "-c" , "copy" , path ] ) ;
3354+ return path ;
3355+ } ;
3356+ const pq = await segment ( "pq" , 16 ) ;
3357+ const mixed = await mux ( "trim-hlg-then-pq" , Buffer . concat ( [ await segment ( "hlg" , 18 ) , pq ] ) ) ;
3358+ const pqOnly = await mux ( "trim-pq-only" , pq ) ;
3359+ const trimmed = join ( fixtureDir , "trim-from-hlg.mp4" ) ;
3360+ await run ( [ "-ss" , "1" , "-i" , mixed , "-c" , "copy" , trimmed ] ) ;
3361+ const extract = ( source : string , id : string , start : number ) =>
3362+ extractVideoFramesRange ( source , id , start , 0.04 , {
3363+ fps : 25 ,
3364+ outputDir : join ( fixtureDir , `out-${ id } ` ) ,
3365+ format : "png" ,
3366+ toneMapHdrToSdr : true ,
3367+ } ) ;
3368+
3369+ // Trimmed 1.6 s is the mixed file's 2.6 s: the PQ segment's 0.6 s.
3370+ const fromTrim = await extract ( trimmed , "trim-from-hlg" , 1.6 ) ;
3371+ const fromPq = await extract ( pqOnly , "trim-pq-only" , 0.6 ) ;
3372+
3373+ const shown = readFirstFramePixel ( fromTrim . framePaths . get ( 0 ) ! , 40 , 40 ) ;
3374+ const expected = readFirstFramePixel ( fromPq . framePaths . get ( 0 ) ! , 40 , 40 ) ;
3375+ expect ( Math . max ( ...shown . map ( ( v , i ) => Math . abs ( v - expected [ i ] ! ) ) ) ) . toBeLessThanOrEqual ( 2 ) ;
3376+ } , 60_000 ) ;
3377+
3378+ it ( "matches Studio's HLG tone map and isolates transformed cache entries" , async ( ) => {
3379+ const source = join ( fixtureDir , "hlg-warm.mp4" ) ;
3380+ await synthesizeHlgClip ( source ) ;
31843381
31853382 const reference = join ( fixtureDir , "studio-reference.png" ) ;
31863383 const referenceResult = await runFfmpeg ( [
@@ -3243,9 +3440,107 @@ describe.skipIf(!HAS_FFMPEG || process.platform === "darwin")("forced-SDR HDR ex
32433440 if ( ! path ) throw new Error ( "expected extracted frame" ) ;
32443441 return readFileSync ( path ) ;
32453442 } ;
3246- expect ( frame ( toneMapped ) ) . toEqual ( readFileSync ( reference ) ) ;
3443+ // Same pixels as Studio's tone map, declared as sRGB so Chrome shows them unconverted.
3444+ const rgb = ( path : string ) : Buffer =>
3445+ spawnSync ( "ffmpeg" , [ "-v" , "error" , "-i" , path , "-f" , "rawvideo" , "-pix_fmt" , "rgb24" , "-" ] )
3446+ . stdout ;
3447+ const toneMappedPath = toneMapped . extracted [ 0 ] ! . framePaths . get ( 0 ) ! ;
3448+ expect ( rgb ( toneMappedPath ) ) . toEqual ( rgb ( reference ) ) ;
3449+ expect ( pngChunkTypes ( toneMappedPath ) ) . toContain ( "sRGB" ) ;
3450+ expect ( pngChunkTypes ( toneMappedPath ) ) . not . toContain ( "cICP" ) ;
32473451 expect ( frame ( plain ) ) . not . toEqual ( frame ( toneMapped ) ) ;
3452+
3453+ const toneMappedJpg = await extractAllVideoFrames ( [ video ( "tone-mapped-jpg" ) ] , fixtureDir , {
3454+ fps : 1 ,
3455+ outputDir : join ( fixtureDir , "tone-mapped-jpg" ) ,
3456+ format : "jpg" ,
3457+ toneMapHdrToSdr : true ,
3458+ } ) ;
3459+ expect ( toneMappedJpg . errors ) . toEqual ( [ ] ) ;
3460+ const jpgPixels = rgb ( toneMappedJpg . extracted [ 0 ] ! . framePaths . get ( 0 ) ! ) ;
3461+ const referencePixels = rgb ( reference ) ;
3462+ const worst = Math . max ( ...[ ...jpgPixels ] . map ( ( v , i ) => Math . abs ( v - referencePixels [ i ] ! ) ) ) ;
3463+ expect ( worst , "tone-mapped jpg against Studio's tone map" ) . toBeLessThanOrEqual ( 3 ) ;
32483464 expect ( frame ( toneMappedAgain ) ) . toEqual ( frame ( toneMapped ) ) ;
3465+
3466+ // A macOS ffmpeg that has zscale uses the same tone map, not VideoToolbox.
3467+ const platform = Object . getOwnPropertyDescriptor ( process , "platform" ) ! ;
3468+ Object . defineProperty ( process , "platform" , { value : "darwin" } ) ;
3469+ try {
3470+ const mac = await extractAllVideoFrames ( [ video ( "tone-mapped-mac" ) ] , fixtureDir , {
3471+ fps : 1 ,
3472+ outputDir : join ( fixtureDir , "tone-mapped-mac" ) ,
3473+ format : "png" ,
3474+ toneMapHdrToSdr : true ,
3475+ } ) ;
3476+ expect ( mac . errors ) . toEqual ( [ ] ) ;
3477+ expect ( rgb ( mac . extracted [ 0 ] ! . framePaths . get ( 0 ) ! ) ) . toEqual ( rgb ( reference ) ) ;
3478+ } finally {
3479+ Object . defineProperty ( process , "platform" , platform ) ;
3480+ }
3481+ } , 60_000 ) ;
3482+
3483+ it ( "warns once and keeps VideoToolbox, under its own cache key, when a macOS ffmpeg has no zscale" , async ( ) => {
3484+ const source = join ( fixtureDir , "hlg-no-zscale.mp4" ) ;
3485+ await synthesizeHlgClip ( source ) ;
3486+ const cacheDir = join ( fixtureDir , "no-zscale-cache" ) ;
3487+ const clip = ( id : string ) : VideoElement => ( {
3488+ id,
3489+ src : source ,
3490+ start : 0 ,
3491+ end : 1 ,
3492+ mediaStart : 0 ,
3493+ loop : false ,
3494+ hasAudio : false ,
3495+ } ) ;
3496+ const options = ( id : string ) => ( {
3497+ fps : 1 ,
3498+ outputDir : join ( fixtureDir , id ) ,
3499+ format : "png" as const ,
3500+ toneMapHdrToSdr : true ,
3501+ } ) ;
3502+ const zscale = await extractAllVideoFrames (
3503+ [ clip ( "zscale" ) ] ,
3504+ fixtureDir ,
3505+ options ( "zscale" ) ,
3506+ undefined ,
3507+ {
3508+ extractCacheDir : cacheDir ,
3509+ } ,
3510+ ) ;
3511+ expect ( zscale . errors ) . toEqual ( [ ] ) ;
3512+
3513+ const realPlatform = process . platform ;
3514+ vi . resetModules ( ) ;
3515+ vi . doMock ( "../utils/psnrFilterAvailability.js" , ( ) => ( {
3516+ isFfmpegFilterAvailable : async ( ) => false ,
3517+ isPsnrFilterAvailable : async ( ) => false ,
3518+ } ) ) ;
3519+ const platform = Object . getOwnPropertyDescriptor ( process , "platform" ) ! ;
3520+ Object . defineProperty ( process , "platform" , { value : "darwin" } ) ;
3521+ const stderr = vi . spyOn ( process . stderr , "write" ) . mockImplementation ( ( ) => true ) ;
3522+ try {
3523+ const { extractAllVideoFrames : extractOnMac } = await import ( "./videoFrameExtractor.js" ) ;
3524+ for ( const [ i , id ] of [ "no-zscale-1" , "no-zscale-2" ] . entries ( ) ) {
3525+ const result = await extractOnMac ( [ clip ( id ) ] , fixtureDir , options ( id ) , undefined , {
3526+ extractCacheDir : cacheDir ,
3527+ } ) ;
3528+ // The zscale frames above must not be served for the VideoToolbox path.
3529+ if ( i === 0 ) expect ( result . phaseBreakdown . cacheHits ) . toBe ( 0 ) ;
3530+ // Off macOS the VideoToolbox decode itself fails, which proves it was attempted.
3531+ if ( realPlatform !== "darwin" )
3532+ expect ( JSON . stringify ( result . errors ) ) . toMatch ( / v i d e o t o o l b o x / i) ;
3533+ }
3534+ const zscaleWarnings = stderr . mock . calls . filter ( ( [ message ] ) =>
3535+ String ( message ) . includes ( "no zscale filter" ) ,
3536+ ) ;
3537+ expect ( zscaleWarnings ) . toHaveLength ( 1 ) ;
3538+ } finally {
3539+ stderr . mockRestore ( ) ;
3540+ Object . defineProperty ( process , "platform" , platform ) ;
3541+ vi . doUnmock ( "../utils/psnrFilterAvailability.js" ) ;
3542+ vi . resetModules ( ) ;
3543+ }
32493544 } , 60_000 ) ;
32503545
32513546 // A second process would read the frames back without the HDR10 light-level metadata the tone map uses.
@@ -3283,7 +3578,7 @@ describe.skipIf(!HAS_FFMPEG || process.platform === "darwin")("forced-SDR HDR ex
32833578 "-t" ,
32843579 "0.616666" ,
32853580 "-vf" ,
3286- "zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv" ,
3581+ "zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv,setparams=color_primaries=bt709:color_trc=iec61966-2-1 " ,
32873582 "-fps_mode" ,
32883583 "cfr" ,
32893584 "-r" ,
0 commit comments