Skip to content

Commit cffc3aa

Browse files
feat(render): report encoding and assembling progress (#5137)
* feat(render): report encoding and assembling progress * fix(render): close every progress stage and fail a render whose progress callback throws * fix(render): send a repeated progress update once * fix(render): keep Docker terminal output as before and drop the unneeded callback ending * fix(render): keep one failure stage code per stage whatever its frame counts
1 parent 891a18a commit cffc3aa

25 files changed

Lines changed: 553 additions & 53 deletions

‎packages/cli/src/commands/render.test.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2047,6 +2047,16 @@ describe("normalizeStageCode", () => {
20472047
expect(normalizeStageCode("Starting browsers (5/6 ready)")).toBe("starting_browsers");
20482048
});
20492049

2050+
it("keeps one code per stage whatever its live frame counts", () => {
2051+
expect(normalizeStageCode("Encoding frame 600/600")).toBe("encoding_video");
2052+
expect(normalizeStageCode("Encoding frame 12/90")).toBe("encoding_video");
2053+
expect(normalizeStageCode("Capturing frame 120/600 (6 workers)")).toBe("capturing_frame");
2054+
expect(normalizeStageCode("Streaming frame 3/40 (segment 1/2, 2 workers)")).toBe(
2055+
"streaming_frame",
2056+
);
2057+
expect(normalizeStageCode("Assembling final video")).toBe("assembling_final_video");
2058+
});
2059+
20502060
it("slugifies an unrecognized stage string instead of bucketing it as unknown", () => {
20512061
expect(normalizeStageCode("Some New Stage!")).toBe("some_new_stage");
20522062
});

‎packages/cli/src/commands/render.ts‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ import {
7272
errorBox,
7373
} from "../ui/format.js";
7474
import { warnIfWebmAlphaDropped } from "../utils/webmAlphaCheck.js";
75-
import { renderProgress } from "../ui/progress.js";
75+
import { renderProgress, renderMachineProgress } from "../ui/progress.js";
7676
import {
7777
trackRenderComplete,
7878
trackRenderError,
@@ -801,6 +801,7 @@ async function renderDocker(
801801
outputDir: resolve(outputDir),
802802
outputFilename,
803803
platform,
804+
hostStdoutIsTty: process.stdout.isTTY === true,
804805
options: {
805806
fps: options.fps,
806807
quality: options.quality,
@@ -1095,8 +1096,9 @@ async function executeLocalRender(
10951096

10961097
const onProgress = options.quiet
10971098
? undefined
1098-
: (progressJob: { progress: number }, message: string) => {
1099+
: (progressJob: Pick<RenderJob, "progress" | "stageProgress">, message: string) => {
10991100
renderProgress(progressJob.progress, message);
1101+
renderMachineProgress(progressJob.progress, progressJob.stageProgress);
11001102
};
11011103

11021104
try {
@@ -1654,15 +1656,19 @@ const KNOWN_STAGE_CODES: Readonly<Record<string, string>> = {
16541656
"Render complete": "render_complete",
16551657
"Render cancelled": "render_cancelled",
16561658
pipeline: "pipeline",
1659+
"Starting browsers": "starting_browsers",
1660+
// Was "Encoding video" before encode reported frames; keeps the same bucket.
1661+
"Encoding frame": "encoding_video",
16571662
};
16581663

1664+
// Live counts in a progress label ("Capturing frame 120/600 (6 workers)") would make a code per render.
1665+
const STAGE_COUNTS = /\([^)]*\)|\d+\/\d+/g;
1666+
16591667
export function normalizeStageCode(stage: string): string {
1660-
const known = KNOWN_STAGE_CODES[stage];
1668+
const base = stage.replace(STAGE_COUNTS, "").trim();
1669+
const known = KNOWN_STAGE_CODES[base];
16611670
if (known) return known;
1662-
// The producer's "Starting browsers (k/n ready)" carries live counts; keep one code for it.
1663-
if (stage.startsWith("Starting browsers")) return "starting_browsers";
1664-
const slug = stage
1665-
.trim()
1671+
const slug = base
16661672
.toLowerCase()
16671673
.replace(/[^a-z0-9]+/g, "_")
16681674
.replace(/^_+|_+$/g, "");

‎packages/cli/src/ui/progress.test.ts‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { afterEach, describe, expect, it } from "vitest";
22

3-
import { renderProgress } from "./progress.js";
3+
import { renderMachineProgress, renderProgress } from "./progress.js";
44

55
const originalWrite = process.stdout.write.bind(process.stdout);
66
const originalIsTTY = process.stdout.isTTY;
@@ -31,3 +31,35 @@ describe("renderProgress", () => {
3131
expect(output).not.toContain("\r");
3232
});
3333
});
34+
35+
describe("renderMachineProgress", () => {
36+
const capture = (isTTY: boolean) => {
37+
let output = "";
38+
Object.defineProperty(process.stdout, "isTTY", { value: isTTY, configurable: true });
39+
process.stdout.write = ((chunk: string | Uint8Array) => {
40+
output += String(chunk);
41+
return true;
42+
}) as typeof process.stdout.write;
43+
renderMachineProgress(83.4, { code: "encode", done: 812, total: 1800 });
44+
return output;
45+
};
46+
47+
it("writes one @hf-progress line for a parent reading piped stdout", () => {
48+
expect(capture(false)).toBe(
49+
'@hf-progress {"code":"encode","done":812,"total":1800,"pct":83}\n',
50+
);
51+
});
52+
53+
it("stays silent in a terminal", () => {
54+
expect(capture(true)).toBe("");
55+
});
56+
57+
it("stays silent in a container whose host prints to a terminal", () => {
58+
process.env.HYPERFRAMES_STDOUT_IS_TTY = "1";
59+
try {
60+
expect(capture(false)).toBe("");
61+
} finally {
62+
delete process.env.HYPERFRAMES_STDOUT_IS_TTY;
63+
}
64+
});
65+
});

‎packages/cli/src/ui/progress.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { c } from "./colors.js";
2+
import type { RenderJob } from "@hyperframes/producer";
23

34
const { stdout } = process;
45

@@ -18,3 +19,8 @@ export function renderProgress(percent: number, stage: string, row?: number): vo
1819
stdout.write(`\r\x1b[2K${line}`);
1920
}
2021
}
22+
23+
export function renderMachineProgress(percent: number, stage: RenderJob["stageProgress"]): void {
24+
if (stdout.isTTY || process.env.HYPERFRAMES_STDOUT_IS_TTY === "1" || !stage) return;
25+
stdout.write(`@hf-progress ${JSON.stringify({ ...stage, pct: Math.round(percent) })}\n`);
26+
}

‎packages/cli/src/utils/dockerRunArgs.test.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,14 @@ describe("buildDockerRunArgs", () => {
5656
`);
5757
});
5858

59+
it("tells the container its output reaches a terminal only when the host's stdout is one", () => {
60+
const flags = (hostStdoutIsTty?: boolean) =>
61+
buildDockerRunArgs({ ...FIXED_INPUT, hostStdoutIsTty, options: BASE }).slice(0, 4);
62+
expect(flags(true)).toEqual(["run", "--rm", "-e", "HYPERFRAMES_STDOUT_IS_TTY=1"]);
63+
expect(flags(false)).toEqual(["run", "--rm", "--platform", "linux/amd64"]);
64+
expect(flags(undefined)).toEqual(["run", "--rm", "--platform", "linux/amd64"]);
65+
});
66+
5967
it("omits --workers when auto sizing should happen inside the container", () => {
6068
const args = buildDockerRunArgs({ ...FIXED_INPUT, options: BASE });
6169
expect(args).not.toContain("--workers");

‎packages/cli/src/utils/dockerRunArgs.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ export interface DockerRunArgsInput {
1717
outputDir: string;
1818
/** Filename within `outputDir` (joined to /output inside the container). */
1919
outputFilename: string;
20+
hostStdoutIsTty?: boolean;
2021
/**
2122
* Docker `--platform` value (`linux/amd64` or `linux/arm64`). When omitted,
2223
* resolves to the host architecture via `resolveDockerPlatform()`. Pinning
@@ -109,6 +110,7 @@ export function buildDockerRunArgs(input: DockerRunArgsInput): string[] {
109110
return [
110111
"run",
111112
"--rm",
113+
...(input.hostStdoutIsTty ? ["-e", "HYPERFRAMES_STDOUT_IS_TTY=1"] : []),
112114
"--platform",
113115
platform,
114116
"--shm-size=2g",

‎packages/engine/src/services/chunkEncoder.ts‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,12 @@ import {
3232
import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js";
3333
import { withEvenDimensionPad } from "../utils/evenDimensions.js";
3434
import { SDR_CAPTURE_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js";
35-
import { formatFfmpegError, isExternalFfmpegInterruption, runFfmpeg } from "../utils/runFfmpeg.js";
35+
import {
36+
ffmpegStatsReader,
37+
formatFfmpegError,
38+
isExternalFfmpegInterruption,
39+
runFfmpeg,
40+
} from "../utils/runFfmpeg.js";
3641
import { extractAudioMetadata } from "../utils/ffprobe.js";
3742
import { type Fps, fpsToFfmpegArg, fpsToNumber } from "@hyperframes/core";
3843
import type { EncoderOptions, EncodeResult, MuxResult } from "./chunkEncoder.types.js";
@@ -491,13 +496,19 @@ export function buildEncoderArgs(
491496
return args;
492497
}
493498

499+
const framesReader = (onFrames?: (frames: number) => void) =>
500+
onFrames && ffmpegStatsReader(({ frames }) => frames !== undefined && onFrames(frames));
501+
const secondsReader = (onSeconds?: (seconds: number) => void) =>
502+
onSeconds && ffmpegStatsReader(({ seconds }) => seconds !== undefined && onSeconds(seconds));
503+
494504
export async function encodeFramesFromDir(
495505
framesDir: string,
496506
framePattern: string,
497507
outputPath: string,
498508
options: EncoderOptions,
499509
signal?: AbortSignal,
500510
config?: Partial<Pick<EngineConfig, "ffmpegEncodeTimeout">>,
511+
onFramesEncoded?: (frames: number) => void,
501512
): Promise<EncodeResult> {
502513
const startTime = Date.now();
503514

@@ -527,7 +538,11 @@ export async function encodeFramesFromDir(
527538
const inputArgs = ["-framerate", fpsToFfmpegArg(options.fps), "-i", inputPath];
528539
const args = buildEncoderArgs(options, inputArgs, outputPath, gpuEncoder);
529540
const encodeTimeout = config?.ffmpegEncodeTimeout ?? DEFAULT_CONFIG.ffmpegEncodeTimeout;
530-
const result = await runFfmpeg(args, { signal, timeout: encodeTimeout });
541+
const result = await runFfmpeg(args, {
542+
signal,
543+
timeout: encodeTimeout,
544+
onStderr: framesReader(onFramesEncoded),
545+
});
531546
if (result.terminationReason === "abort") {
532547
return {
533548
success: false,
@@ -636,6 +651,7 @@ export async function encodeFramesChunkedConcat(
636651
chunkSizeFrames: number,
637652
signal?: AbortSignal,
638653
config?: Partial<Pick<EngineConfig, "ffmpegEncodeTimeout">>,
654+
onFramesEncoded?: (frames: number) => void,
639655
): Promise<EncodeResult> {
640656
const start = Date.now();
641657
const files = readdirSync(framesDir)
@@ -693,7 +709,13 @@ export async function encodeFramesChunkedConcat(
693709
if (options.useGpu) gpuEncoder = await getCachedGpuEncoder();
694710
const args = buildEncoderArgs(options, inputArgs, chunkPath, gpuEncoder);
695711
const encodeTimeout = config?.ffmpegEncodeTimeout ?? DEFAULT_CONFIG.ffmpegEncodeTimeout;
696-
const processResult = await runFfmpeg(args, { signal, timeout: encodeTimeout });
712+
const processResult = await runFfmpeg(args, {
713+
signal,
714+
timeout: encodeTimeout,
715+
onStderr: framesReader(
716+
onFramesEncoded && ((frames) => onFramesEncoded(startNumber + frames)),
717+
),
718+
});
697719
const chunkResult = {
698720
success: processResult.success,
699721
error: processResult.success
@@ -750,6 +772,7 @@ export async function muxVideoWithAudio(
750772
signal?: AbortSignal,
751773
config?: MuxVideoWithAudioOptions,
752774
fps?: Fps,
775+
onSecondsWritten?: (seconds: number) => void,
753776
): Promise<MuxResult> {
754777
const outputDir = dirname(outputPath);
755778
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
@@ -804,7 +827,11 @@ export async function muxVideoWithAudio(
804827
args.push("-y", outputPath);
805828

806829
const processTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
807-
const result = await runFfmpeg(args, { signal, timeout: processTimeout });
830+
const result = await runFfmpeg(args, {
831+
signal,
832+
timeout: processTimeout,
833+
onStderr: secondsReader(onSecondsWritten),
834+
});
808835

809836
if (signal?.aborted) {
810837
return {
@@ -977,6 +1004,7 @@ export async function applyFaststart(
9771004
signal?: AbortSignal,
9781005
config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
9791006
fps?: Fps,
1007+
onSecondsWritten?: (seconds: number) => void,
9801008
): Promise<MuxResult> {
9811009
// faststart is MP4-only (moves moov atom to file start for streaming).
9821010
// WebM and MOV don't need it — skip the re-mux.
@@ -995,7 +1023,11 @@ export async function applyFaststart(
9951023
args.push("-y", outputPath);
9961024

9971025
const processTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
998-
const result = await runFfmpeg(args, { signal, timeout: processTimeout });
1026+
const result = await runFfmpeg(args, {
1027+
signal,
1028+
timeout: processTimeout,
1029+
onStderr: secondsReader(onSecondsWritten),
1030+
});
9991031

10001032
if (signal?.aborted) {
10011033
return {

‎packages/engine/src/services/streamingEncoder.test.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -604,6 +604,27 @@ describe("spawnStreamingEncoder lifecycle and cleanup", () => {
604604
expect(result.fileSize).toBe(0); // No real ffmpeg, no file written
605605
});
606606

607+
it("reports ffmpeg's encoded-frame count while close() waits, and stops at exit", async () => {
608+
const { spawn, calls } = createSpawnSpy();
609+
vi.resetModules();
610+
vi.doMock("child_process", () => ({ spawn }));
611+
612+
const { spawnStreamingEncoder } = await import("./streamingEncoder.js");
613+
const dir = mkdtempSync(join(tmpdir(), "se-encoded-"));
614+
const encoder = await spawnStreamingEncoder(join(dir, "out.mp4"), baseOptions);
615+
const proc = calls[0]!.proc;
616+
proc.stderr.emit("data", Buffer.from("frame= 10 fps=20 q=28.0 size=1kB time=00:00:00.33\r"));
617+
618+
const heard: number[] = [];
619+
const closePromise = encoder.close((frames) => heard.push(frames));
620+
proc.stderr.emit("data", Buffer.from("frame= 25 fps=20 q=28.0 size=2kB time=00:00:00.83\r"));
621+
process.nextTick(() => proc.emit("close", 0));
622+
await closePromise;
623+
proc.stderr.emit("data", Buffer.from("frame= 30 fps=20 q=28.0 size=2kB time=00:00:01.00\r"));
624+
625+
expect(heard).toEqual([10, 25]);
626+
});
627+
607628
it("returns a failure result (does NOT throw) when ffmpeg exits non-zero before close()", async () => {
608629
const { spawn, calls } = createSpawnSpy();
609630
vi.resetModules();

‎packages/engine/src/services/streamingEncoder.ts‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,11 @@ import {
3131
buildVideoToolboxRateControlArgs,
3232
mapPresetForGpuEncoder,
3333
} from "../utils/gpuEncoder.js";
34-
import { formatFfmpegError, isExternalFfmpegInterruption } from "../utils/runFfmpeg.js";
34+
import {
35+
ffmpegStatsReader,
36+
formatFfmpegError,
37+
isExternalFfmpegInterruption,
38+
} from "../utils/runFfmpeg.js";
3539
import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
3640
import { getHdrEncoderColorParams } from "../utils/hdr.js";
3741
import { withEvenDimensionPad } from "../utils/evenDimensions.js";
@@ -187,7 +191,8 @@ export interface StreamingEncoder {
187191
* calls would interleave frame bytes on the pipe and race the drain wait.
188192
*/
189193
writeFrame: (buffer: Buffer) => Promise<boolean>;
190-
close: () => Promise<StreamingEncoderResult>;
194+
/** `onFramesEncoded` hears ffmpeg's encoded-frame count while close waits for it to finish. */
195+
close: (onFramesEncoded?: (frames: number) => void) => Promise<StreamingEncoderResult>;
191196
getExitStatus: () => "running" | "success" | "error";
192197
/**
193198
* The FFmpeg failure reason (exit code + tail of stderr), or `undefined`
@@ -497,9 +502,16 @@ export async function spawnStreamingEncoder(
497502
// libx264 printed its summary and exited 255, observable as
498503
// "Streaming encode failed: FFmpeg exited with code 255" with audio:0kB).
499504
const streamingTimeout = config?.ffmpegStreamingTimeout ?? DEFAULT_CONFIG.ffmpegStreamingTimeout;
505+
let framesEncoded = 0;
506+
let onFramesEncoded: ((frames: number) => void) | undefined;
500507
const managed = new ManagedChildProcess(ffmpeg, {
501508
signal,
502509
inactivityTimeoutMs: streamingTimeout,
510+
onStderr: ffmpegStatsReader(({ frames }) => {
511+
if (frames === undefined) return;
512+
framesEncoded = frames;
513+
onFramesEncoded?.(frames);
514+
}),
503515
});
504516
const exitPromise = managed.wait().then((outcome) => {
505517
exitCode = outcome.exitCode;
@@ -604,7 +616,7 @@ export async function spawnStreamingEncoder(
604616
return true;
605617
},
606618

607-
close: async (): Promise<StreamingEncoderResult> => {
619+
close: async (onEncoded?: (frames: number) => void): Promise<StreamingEncoderResult> => {
608620
// INVARIANT: close() is idempotent. The renderOrchestrator HDR cleanup
609621
// path tracks an `encoderClosed` flag and may still re-call close() in
610622
// the outer finally if the inner cleanup raised before the flag flipped.
@@ -616,6 +628,10 @@ export async function spawnStreamingEncoder(
616628
// repeated calls. If you change this method, preserve idempotency or
617629
// a regression here will silently double-close ffmpeg and produce
618630
// harder-to-trace errors at the orchestrator layer.
631+
if (onEncoded && exitStatus === "running") {
632+
onFramesEncoded = onEncoded;
633+
onEncoded(framesEncoded);
634+
}
619635
const stdin = ffmpeg.stdin;
620636
if (stdin && !stdin.destroyed) {
621637
await new Promise<void>((resolve) => {

0 commit comments

Comments
 (0)