Skip to content

Commit 0f7eebd

Browse files
authored
fix(encoder): signal host interruptions for retry (#3578)
* fix(encoder): signal host interruptions for retry * fix(encoder): cover all render interruption paths * fix(encoder): classify HDR pre-extraction drains
1 parent 9097d53 commit 0f7eebd

35 files changed

Lines changed: 604 additions & 44 deletions

‎packages/engine/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,7 @@ export {
309309
export {
310310
runFfmpeg,
311311
formatFfmpegError,
312+
isExternalFfmpegInterruption,
312313
type RunFfmpegOptions,
313314
type RunFfmpegResult,
314315
} from "./utils/runFfmpeg.js";

‎packages/engine/src/services/audioMixer.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -383,7 +383,10 @@ function ffmpegFailure(
383383
let owner: AudioProcessingFailure["owner"] = "system";
384384
let retryable = false;
385385

386-
if (result.terminationReason === "abort") {
386+
if (result.failureReason === "external_interruption") {
387+
reason = "external_interruption";
388+
retryable = true;
389+
} else if (result.terminationReason === "abort") {
387390
reason = "cancelled";
388391
owner = "user";
389392
} else if (result.terminationReason === "deadline" || result.terminationReason === "inactivity") {
@@ -862,7 +865,12 @@ async function mixAudioTracks(
862865
// dropped from the output entirely — a missing fade beats missing audio.
863866
let degradedAutomation = false;
864867
const hasAutomation = tracks.some((track) => (track.volumeKeyframes?.length ?? 0) > 0);
865-
if (!result.success && !signal?.aborted && hasAutomation) {
868+
if (
869+
!result.success &&
870+
result.failureReason !== "external_interruption" &&
871+
!signal?.aborted &&
872+
hasAutomation
873+
) {
866874
const retry = await runMix(true);
867875
if (retry.success) {
868876
result = retry;

‎packages/engine/src/services/audioMixer.types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ export type AudioFailureReason =
6969
| "ffmpeg_unsupported"
7070
| "ffmpeg_timeout"
7171
| "ffmpeg_unavailable"
72+
| "external_interruption"
7273
| "ffmpeg_failed"
7374
| "cancelled"
7475
| "internal";

‎packages/engine/src/services/chunkEncoder.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,45 @@ describe("encodeFramesChunkedConcat ffmpegEncodeTimeout", () => {
374374
});
375375

376376
describe("muxVideoWithAudio audio codec handling", () => {
377+
it("preserves an external interruption from mux", async () => {
378+
const { spawn, calls } = createSpawnSpy();
379+
vi.resetModules();
380+
vi.doMock("child_process", () => ({ spawn }));
381+
382+
const { muxVideoWithAudio } = await import("./chunkEncoder.js");
383+
const muxPromise = muxVideoWithAudio(
384+
"/tmp/video-only.mp4",
385+
"/tmp/audio.aac",
386+
"/tmp/output.mp4",
387+
);
388+
389+
await flushMuxCodecResolution();
390+
calls[0]!.proc.stderr.emit("data", Buffer.from("Exiting normally, received signal 15.\n"));
391+
emitClose(calls[0]!.proc, 255);
392+
393+
await expect(muxPromise).resolves.toMatchObject({
394+
success: false,
395+
failureReason: "external_interruption",
396+
});
397+
});
398+
399+
it("preserves an external interruption from faststart", async () => {
400+
const { spawn, calls } = createSpawnSpy();
401+
vi.resetModules();
402+
vi.doMock("child_process", () => ({ spawn }));
403+
404+
const { applyFaststart } = await import("./chunkEncoder.js");
405+
const faststartPromise = applyFaststart("/tmp/video-only.mp4", "/tmp/output.mp4");
406+
407+
calls[0]!.proc.stderr.emit("data", Buffer.from("Exiting normally, received signal 15.\n"));
408+
emitClose(calls[0]!.proc, 255);
409+
410+
await expect(faststartPromise).resolves.toMatchObject({
411+
success: false,
412+
failureReason: "external_interruption",
413+
});
414+
});
415+
377416
it("copies HyperFrames AAC sidecars into MP4 instead of re-encoding", async () => {
378417
const { spawn, calls } = createSpawnSpy();
379418
vi.resetModules();

‎packages/engine/src/services/chunkEncoder.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import {
1717
} from "../utils/gpuEncoder.js";
1818
import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js";
1919
import { withEvenDimensionPad } from "../utils/evenDimensions.js";
20-
import { formatFfmpegError, runFfmpeg } from "../utils/runFfmpeg.js";
20+
import { formatFfmpegError, isExternalFfmpegInterruption, runFfmpeg } from "../utils/runFfmpeg.js";
2121
import { extractAudioMetadata } from "../utils/ffprobe.js";
2222
import { type Fps, fpsToFfmpegArg } from "@hyperframes/core";
2323
import type { EncoderOptions, EncodeResult, MuxResult } from "./chunkEncoder.types.js";
@@ -518,6 +518,7 @@ export async function encodeFramesFromDir(
518518
result.terminationReason === "deadline",
519519
encodeTimeout,
520520
),
521+
failureReason: isExternalFfmpegInterruption(result) ? "external_interruption" : undefined,
521522
};
522523
}
523524
const fileSize = existsSync(outputPath) ? statSync(outputPath).size : 0;
@@ -612,6 +613,9 @@ export async function encodeFramesChunkedConcat(
612613
framesEncoded: 0,
613614
fileSize: 0,
614615
error: chunkResult.error,
616+
failureReason: isExternalFfmpegInterruption(processResult)
617+
? "external_interruption"
618+
: undefined,
615619
};
616620
}
617621
chunkPaths.push(chunkPath);
@@ -650,6 +654,9 @@ export async function encodeFramesChunkedConcat(
650654
framesEncoded: 0,
651655
fileSize: 0,
652656
error: concatResult.error,
657+
failureReason: isExternalFfmpegInterruption(concatProcessResult)
658+
? "external_interruption"
659+
: undefined,
653660
};
654661
}
655662

@@ -739,6 +746,7 @@ export async function muxVideoWithAudio(
739746
outputPath,
740747
durationMs: result.durationMs,
741748
error: !result.success ? formatFfmpegError(result.exitCode, result.stderr) : undefined,
749+
failureReason: result.failureReason,
742750
};
743751
}
744752

@@ -781,5 +789,6 @@ export async function applyFaststart(
781789
outputPath,
782790
durationMs: result.durationMs,
783791
error: !result.success ? formatFfmpegError(result.exitCode, result.stderr) : undefined,
792+
failureReason: result.failureReason,
784793
};
785794
}

‎packages/engine/src/services/chunkEncoder.types.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,15 @@ export interface EncodeResult {
5050
framesEncoded: number;
5151
fileSize: number;
5252
error?: string;
53+
/** Stable machine-readable cause for failures safe to retry on a fresh host. */
54+
failureReason?: "external_interruption";
5355
}
5456

5557
export interface MuxResult {
5658
success: boolean;
5759
outputPath: string;
5860
durationMs: number;
5961
error?: string;
62+
/** Stable machine-readable cause for failures safe to retry on a fresh host. */
63+
failureReason?: "external_interruption";
6064
}

‎packages/engine/src/services/streamingEncoder.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -548,6 +548,24 @@ describe("spawnStreamingEncoder lifecycle and cleanup", () => {
548548
expect(result.error).toContain("Encoder error");
549549
});
550550

551+
it("classifies ffmpeg's handled SIGTERM as an external interruption", async () => {
552+
const { spawn, calls } = createSpawnSpy();
553+
vi.resetModules();
554+
vi.doMock("child_process", () => ({ spawn }));
555+
556+
const { spawnStreamingEncoder } = await import("./streamingEncoder.js");
557+
const dir = mkdtempSync(join(tmpdir(), "se-interrupted-"));
558+
const encoder = await spawnStreamingEncoder(join(dir, "out.mp4"), baseOptions);
559+
const proc = calls[0]!.proc;
560+
proc.stderr.emit("data", Buffer.from("Exiting normally, received signal 15.\n"));
561+
process.nextTick(() => proc.emit("close", 255));
562+
563+
const result = await encoder.close();
564+
expect(result.success).toBe(false);
565+
expect(result.failureReason).toBe("external_interruption");
566+
expect(encoder.getExitFailureReason?.()).toBe("external_interruption");
567+
});
568+
551569
it("getExitError surfaces the ffmpeg failure reason after a non-zero exit", async () => {
552570
const { spawn, calls } = createSpawnSpy();
553571
vi.resetModules();
@@ -696,6 +714,27 @@ describe("spawnStreamingEncoder lifecycle and cleanup", () => {
696714
expect(await encoder.writeFrame(Buffer.from([0]))).toBe(false);
697715
});
698716

717+
it("waits for child close when stdin dies first so the interruption reason is observable", async () => {
718+
const { spawn, calls } = createSpawnSpy();
719+
vi.resetModules();
720+
vi.doMock("child_process", () => ({ spawn }));
721+
722+
const { spawnStreamingEncoder } = await import("./streamingEncoder.js");
723+
const dir = mkdtempSync(join(tmpdir(), "se-epipe-before-close-"));
724+
const encoder = await spawnStreamingEncoder(join(dir, "out.mp4"), baseOptions);
725+
const proc = calls[0]!.proc;
726+
proc.stdin.destroyed = true;
727+
728+
const writePromise = encoder.writeFrame(Buffer.from([0]));
729+
await expect(resolveWithin(writePromise, 10)).resolves.toBe("timeout");
730+
731+
proc.stderr.emit("data", Buffer.from("Exiting normally, received signal 15.\n"));
732+
proc.emit("close", 255);
733+
734+
await expect(writePromise).resolves.toBe(false);
735+
expect(encoder.getExitFailureReason?.()).toBe("external_interruption");
736+
});
737+
699738
it("writeFrame waits for stdin drain when FFmpeg applies back-pressure", async () => {
700739
const { spawn, calls } = createSpawnSpy();
701740
vi.resetModules();

‎packages/engine/src/services/streamingEncoder.ts‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import {
2929
getGpuEncoderName,
3030
mapPresetForGpuEncoder,
3131
} from "../utils/gpuEncoder.js";
32-
import { formatFfmpegError } from "../utils/runFfmpeg.js";
32+
import { formatFfmpegError, isExternalFfmpegInterruption } from "../utils/runFfmpeg.js";
3333
import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
3434
import { getHdrEncoderColorParams } from "../utils/hdr.js";
3535
import { withEvenDimensionPad } from "../utils/evenDimensions.js";
@@ -159,6 +159,8 @@ export interface StreamingEncoderResult {
159159
durationMs: number;
160160
fileSize: number;
161161
error?: string;
162+
/** Stable machine-readable cause for failures safe to retry on a fresh host. */
163+
failureReason?: "external_interruption";
162164
}
163165

164166
export interface StreamingEncoder {
@@ -179,6 +181,8 @@ export interface StreamingEncoder {
179181
* unsupported codec, disk full) instead of a bare "encoder exited" message.
180182
*/
181183
getExitError: () => string | undefined;
184+
/** Machine-readable cause available after FFmpeg exits unexpectedly mid-write. */
185+
getExitFailureReason?: () => "external_interruption" | undefined;
182186
}
183187

184188
/**
@@ -465,6 +469,7 @@ export async function spawnStreamingEncoder(
465469
let exitStatus: "running" | "success" | "error" = "running";
466470
let stderr = "";
467471
let exitCode: number | null = null;
472+
let exitSignal: NodeJS.Signals | null = null;
468473
let terminationReason: ManagedProcessTerminationReason = "exit";
469474

470475
ffmpeg.stdin?.on("error", () => {});
@@ -486,6 +491,7 @@ export async function spawnStreamingEncoder(
486491
});
487492
const exitPromise = managed.wait().then((outcome) => {
488493
exitCode = outcome.exitCode;
494+
exitSignal = outcome.signal;
489495
stderr = outcome.stderr;
490496
terminationReason = outcome.reason;
491497
exitStatus = outcome.reason === "exit" && outcome.exitCode === 0 ? "success" : "error";
@@ -530,7 +536,15 @@ export async function spawnStreamingEncoder(
530536
const encoder: StreamingEncoder = {
531537
writeFrame: async (buffer: Buffer): Promise<boolean> => {
532538
const stdin = ffmpeg.stdin;
533-
if (exitStatus !== "running" || !stdin || stdin.destroyed) {
539+
if (exitStatus !== "running") {
540+
return false;
541+
}
542+
if (!stdin || stdin.destroyed) {
543+
// The OS can close the pipe (EPIPE) before Node delivers the child
544+
// process `close` event. Wait for the shared exit settlement so the
545+
// caller can synchronously inspect getExitFailureReason() instead of
546+
// losing a host-interruption signal in this narrow race.
547+
await exitPromise;
534548
return false;
535549
}
536550
// Copy the buffer before writing — Node streams hold a reference to the
@@ -604,6 +618,14 @@ export async function spawnStreamingEncoder(
604618
durationMs,
605619
fileSize: 0,
606620
error: `${formatFfmpegError(exitCode, stderr)}${inactivitySuffix}`,
621+
failureReason: isExternalFfmpegInterruption({
622+
exitCode,
623+
signal: exitSignal,
624+
stderr,
625+
terminationReason,
626+
})
627+
? "external_interruption"
628+
: undefined,
607629
};
608630
}
609631

@@ -618,6 +640,18 @@ export async function spawnStreamingEncoder(
618640
if (exitStatus !== "error") return undefined;
619641
return formatFfmpegError(exitCode, stderr);
620642
},
643+
644+
getExitFailureReason: () => {
645+
if (exitStatus !== "error") return undefined;
646+
return isExternalFfmpegInterruption({
647+
exitCode,
648+
signal: exitSignal,
649+
stderr,
650+
terminationReason,
651+
})
652+
? "external_interruption"
653+
: undefined;
654+
},
621655
};
622656

623657
return encoder;

‎packages/engine/src/services/videoFrameExtractor.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,7 @@ export interface ExtractionPhaseBreakdown {
260260

261261
export type VideoExtractionFailureKind =
262262
| "cancelled"
263+
| "external_interruption"
263264
| "source_missing"
264265
| "source_rejected"
265266
| "download_not_found"
@@ -785,6 +786,14 @@ export async function extractVideoFramesRange(
785786
args.push("-y", outputPattern);
786787

787788
const processResult = await runFfmpeg(args, { signal, timeout: ffmpegProcessTimeout });
789+
if (processResult.failureReason === "external_interruption") {
790+
throw new VideoSourceExtractionError(
791+
"external_interruption",
792+
true,
793+
"Video frame extraction interrupted by host lifecycle",
794+
`FFmpeg exited with code ${processResult.exitCode}: ${processResult.stderr.slice(-500)}`,
795+
);
796+
}
788797
if (processResult.terminationReason === "abort") {
789798
throw new VideoSourceExtractionError("cancelled", false, "Video extraction cancelled");
790799
}

‎packages/engine/src/utils/runFfmpeg.test.ts‎

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,65 @@ import { EventEmitter } from "node:events";
22
import { resolve } from "node:path";
33
import { afterEach, describe, expect, it, vi } from "vitest";
44

5-
import { formatFfmpegError } from "./runFfmpeg.js";
5+
import { formatFfmpegError, isExternalFfmpegInterruption } from "./runFfmpeg.js";
6+
7+
describe("isExternalFfmpegInterruption", () => {
8+
const base = {
9+
exitCode: 255,
10+
signal: null,
11+
stderr: "",
12+
terminationReason: "exit" as const,
13+
};
14+
15+
it("recognizes a direct external termination signal", () => {
16+
expect(isExternalFfmpegInterruption({ ...base, exitCode: null, signal: "SIGTERM" })).toBe(true);
17+
});
18+
19+
it("does not broaden the retry signal to SIGKILL", () => {
20+
expect(isExternalFfmpegInterruption({ ...base, exitCode: null, signal: "SIGKILL" })).toBe(
21+
false,
22+
);
23+
});
24+
25+
it("recognizes ffmpeg's handled SIGTERM exit-255 signature", () => {
26+
expect(
27+
isExternalFfmpegInterruption({
28+
...base,
29+
stderr: "frame= 42\nExiting normally, received signal 15.\n",
30+
}),
31+
).toBe(true);
32+
});
33+
34+
it("does not classify an ordinary exit 255", () => {
35+
expect(isExternalFfmpegInterruption({ ...base, stderr: "Encoder initialization failed" })).toBe(
36+
false,
37+
);
38+
});
39+
40+
it("requires exit 255 when classification relies on ffmpeg stderr", () => {
41+
expect(
42+
isExternalFfmpegInterruption({
43+
...base,
44+
exitCode: 1,
45+
stderr: "Exiting normally, received signal 15.",
46+
}),
47+
).toBe(false);
48+
});
49+
50+
it.each(["abort", "deadline", "inactivity"] as const)(
51+
"keeps a managed %s termination non-retryable",
52+
(terminationReason) => {
53+
expect(
54+
isExternalFfmpegInterruption({
55+
...base,
56+
signal: "SIGTERM",
57+
stderr: "Exiting normally, received signal 15.",
58+
terminationReason,
59+
}),
60+
).toBe(false);
61+
},
62+
);
63+
});
664

765
describe("formatFfmpegError", () => {
866
const originalPlatform = process.platform;

0 commit comments

Comments
 (0)