Skip to content

Commit 6250d30

Browse files
committed
fix(engine): isolate the observe-only region-shift metric from the PSNR gate
regionShift shared a Promise.all + catch with the enforced psnrDb check: on the streaming drain a shift failure skipped the whole sample, and on the disk path a missing blend filter (not covered by the bootstrap preflight) aborted the render as infrastructure failure. The observe-only metric now runs under its own catch on both paths — log and move on — so the PSNR gate stays exactly as strong as on main while the shift gate is unarmed.
1 parent 3248f02 commit 6250d30

4 files changed

Lines changed: 206 additions & 27 deletions

File tree

‎packages/engine/src/services/parallelCoordinator.test.ts‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
import { describe, it, expect, vi } from "vitest";
2+
import { mkdtemp, rm, writeFile } from "fs/promises";
3+
import { tmpdir } from "os";
4+
import { join } from "path";
25
import {
36
assertDiskSampleVerified,
47
calculateOptimalWorkers,
@@ -10,6 +13,7 @@ import {
1013
formatWorkerFailure,
1114
isFfmpegInfrastructureFailure,
1215
isPoolFatalWorkerFailure,
16+
metricsForDiskSample,
1317
selectVerifySampleIndicesForTask,
1418
selectWorkerDiagnostics,
1519
shouldDisableBrowserPoolForParallelWorker,
@@ -22,6 +26,14 @@ import {
2226
import { getDrawElementVerificationDetails } from "./frameCapture.js";
2327
import type { EngineConfig } from "../config.js";
2428
import { CaptureFailure } from "./captureFailure.js";
29+
import { psnrDb, regionShift } from "../utils/psnr.js";
30+
31+
vi.mock("../utils/psnr.js", () => ({
32+
psnrDb: vi.fn(async () => 44),
33+
regionShift: vi.fn(async () => 0),
34+
resolveDeVerifyMinDb: vi.fn(() => 32),
35+
resolveDeVerifyMaxShift: vi.fn(() => undefined),
36+
}));
2537

2638
describe("parallel worker phase deadline", () => {
2739
it("fails a wedged operation with phase and browser diagnostics before the aggregate watchdog", async () => {
@@ -645,7 +657,58 @@ describe("assertDiskSampleVerified", () => {
645657
);
646658
});
647659

660+
it("does not trip an armed shift gate when the shift metric was skipped", () => {
661+
expect(() => assertDiskSampleVerified({ db: 44, shift: undefined }, 32, 3, 7, 1)).not.toThrow();
662+
});
663+
664+
it("still enforces the dB floor when the shift metric was skipped", () => {
665+
expect(() => assertDiskSampleVerified({ db: 20, shift: undefined }, 32, 3, 7, 1)).toThrow(
666+
/20\.0dB < 32dB/,
667+
);
668+
});
669+
648670
it("passes a sample that clears both gates", () => {
649671
expect(() => assertDiskSampleVerified({ db: 48, shift: 2 }, 32, 3, 7, 1)).not.toThrow();
650672
});
651673
});
674+
675+
describe("metricsForDiskSample", () => {
676+
const writeSample = async (): Promise<{ dir: string; framePath: string }> => {
677+
const dir = await mkdtemp(join(tmpdir(), "hf-de-verify-test-"));
678+
const framePath = join(dir, "frame.jpg");
679+
await writeFile(framePath, Buffer.from("captured"));
680+
return { dir, framePath };
681+
};
682+
683+
it("keeps the PSNR result when the region-shift sample fails", async () => {
684+
// The shift metric's ffmpeg chain uses `blend`, which the bootstrap
685+
// preflight doesn't check — a shift-side failure must not forfeit the
686+
// enforced PSNR result or hit the infrastructure abort below.
687+
vi.mocked(regionShift).mockRejectedValueOnce(new Error("No such filter: 'blend'"));
688+
const { dir, framePath } = await writeSample();
689+
try {
690+
expect(await metricsForDiskSample(framePath, Buffer.from("truth"), 1, 7)).toEqual({
691+
db: 44,
692+
shift: undefined,
693+
});
694+
} finally {
695+
await rm(dir, { recursive: true, force: true });
696+
}
697+
});
698+
699+
it("still aborts on a PSNR-side infrastructure failure", async () => {
700+
// The abort moves wholesale → per-side: only the enforced check's own
701+
// ffmpeg failure means the safety net can't run.
702+
vi.mocked(psnrDb).mockRejectedValueOnce(
703+
Object.assign(new Error("spawn ffmpeg ENOENT"), { code: "ENOENT" }),
704+
);
705+
const { dir, framePath } = await writeSample();
706+
try {
707+
await expect(metricsForDiskSample(framePath, Buffer.from("truth"), 1, 7)).rejects.toThrow(
708+
/self-verify aborted/,
709+
);
710+
} finally {
711+
await rm(dir, { recursive: true, force: true });
712+
}
713+
});
714+
});

‎packages/engine/src/services/parallelCoordinator.ts‎

Lines changed: 39 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -721,10 +721,13 @@ function logParDebug(message: () => string): void {
721721
* (low-contrast content missing, issue #3345); log the pass otherwise. Split
722722
* from the sampling loop for the complexity gate.
723723
*
724+
* `metrics.shift` is undefined when the observe-only shift sample failed —
725+
* the enforced PSNR gate still runs, the shift gate is skipped for it.
726+
*
724727
* Exported for testing; the gate decision is a pure metrics read.
725728
*/
726729
export function assertDiskSampleVerified(
727-
metrics: { db: number; shift: number },
730+
metrics: { db: number; shift: number | undefined },
728731
verifyMinDb: number,
729732
verifyMaxShift: number | undefined,
730733
idx: number,
@@ -744,7 +747,11 @@ export function assertDiskSampleVerified(
744747
},
745748
);
746749
}
747-
if (verifyMaxShift !== undefined && metrics.shift > verifyMaxShift) {
750+
if (
751+
verifyMaxShift !== undefined &&
752+
metrics.shift !== undefined &&
753+
metrics.shift > verifyMaxShift
754+
) {
748755
throw new DrawElementVerificationError(
749756
`drawElement self-verify failed at frame ${idx} (disk path, worker ${workerId}): ` +
750757
`region shift ${metrics.shift}/255 > ${verifyMaxShift}/255 vs pre-injection screenshot; ` +
@@ -757,10 +764,12 @@ export function assertDiskSampleVerified(
757764
},
758765
);
759766
}
767+
// The observed max only feeds telemetry on the streaming drain
768+
// (verifyMaxShiftObserved); the disk path logs each sample's shift here.
760769
console.log(
761770
`[Parallel] drawElement disk self-verify passed (worker ${workerId}, frame ${idx}, ` +
762771
`${metrics.db === Infinity ? "inf" : metrics.db.toFixed(1)}dB, ` +
763-
`shift ${metrics.shift}/255)`,
772+
`shift ${metrics.shift === undefined ? "skipped" : `${metrics.shift}/255`})`,
764773
);
765774
}
766775

@@ -788,24 +797,40 @@ export function isFfmpegInfrastructureFailure(err: unknown): boolean {
788797

789798
/**
790799
* Compare one captured frame file against its ground truth. Returns the
791-
* PSNR + worst-region shift, or null on per-sample noise (readFile races,
792-
* transient EPERM, unparseable ffmpeg output on a single sample) — a skipped
793-
* sample is not damage evidence and must not fail the capture. Re-throws when
794-
* the error shape indicates the ffmpeg install itself is broken (missing
795-
* binary or a missing filter): the drawElement self-verify safety net cannot
796-
* possibly run in that state, and continuing would silently ship every
797-
* remaining frame unverified.
800+
* PSNR + worst-region shift — `shift` is undefined when the shift sample
801+
* itself failed — or null on per-sample noise (readFile races, transient
802+
* EPERM, unparseable ffmpeg output on a single sample): a skipped sample is
803+
* not damage evidence and must not fail the capture. Re-throws when the PSNR
804+
* side's error shape indicates the ffmpeg install itself is broken (missing
805+
* binary or a missing `psnr` filter): the drawElement self-verify safety net
806+
* cannot possibly run in that state, and continuing would silently ship
807+
* every remaining frame unverified.
808+
*
809+
* The shift metric is observe-only and its ffmpeg chain uses `blend`, which
810+
* the bootstrap preflight doesn't cover — it runs under its own catch (log
811+
* and move on) so a shift failure can neither forfeit the enforced PSNR
812+
* result nor hit the infrastructure abort. The PSNR gate stays exactly as
813+
* strong as on main.
814+
*
815+
* Exported for testing.
798816
*/
799-
async function metricsForDiskSample(
817+
export async function metricsForDiskSample(
800818
framePath: string,
801819
truth: Buffer,
802820
workerId: number,
803821
idx: number,
804-
): Promise<{ db: number; shift: number } | null> {
822+
): Promise<{ db: number; shift: number | undefined } | null> {
805823
try {
806824
const buf = await readFile(framePath);
807-
const [db, shift] = await Promise.all([psnrDb(buf, truth), regionShift(buf, truth)]);
808-
return { db, shift };
825+
const shiftP = regionShift(buf, truth).catch((err: unknown) => {
826+
console.warn(
827+
`[Parallel] drawElement disk region-shift sample failed (worker ${workerId}, ` +
828+
`frame ${idx}); metric skipped: ${err instanceof Error ? err.message : String(err)}`,
829+
);
830+
return undefined;
831+
});
832+
const db = await psnrDb(buf, truth);
833+
return { db, shift: await shiftP };
809834
} catch (err) {
810835
if (isFfmpegInfrastructureFailure(err)) {
811836
const detail = err instanceof Error ? err.message : String(err);

‎packages/producer/src/services/render/stages/captureStreamingStage.test.ts‎

Lines changed: 74 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ let captureSessionMode: "drawelement" | "screenshot" = "drawelement";
4242
let deVerifyFramesMap: Map<number, Buffer> | undefined;
4343
let psnrDbValue = Infinity;
4444
let regionShiftValue = 0;
45+
let regionShiftError: Error | null = null;
4546
let failPrepareCaptureSessionForReuse = false;
4647
let initializeSessionErrorMessage = "initialize failed";
4748
const browserConsoleBuffer = ["[FrameCapture:ERROR] page.goto failed"];
@@ -195,7 +196,10 @@ mock.module("@hyperframes/engine", () => ({
195196
}
196197
},
197198
psnrDb: async () => psnrDbValue,
198-
regionShift: async () => regionShiftValue,
199+
regionShift: async () => {
200+
if (regionShiftError) throw regionShiftError;
201+
return regionShiftValue;
202+
},
199203
recaptureDrawElementFrameForVerify: async () => Buffer.from("frame"),
200204
spawnStreamingEncoder,
201205
writeCapturedFrame: async () => {},
@@ -772,6 +776,75 @@ describe("runCaptureStreamingStage", () => {
772776
expect(caught).toBeUndefined();
773777
expect(result?.success).toBe(true);
774778
});
779+
780+
it("keeps the enforced PSNR check when the region-shift sample fails", async () => {
781+
// The shift metric's ffmpeg chain uses `blend`, which the bootstrap
782+
// preflight doesn't cover — a shift failure must not forfeit the PSNR
783+
// result (the sample used to be skipped wholesale, letting the render
784+
// pass below the dB floor unverified).
785+
sessionWorkerEncodeEnabled = true;
786+
deVerifyFramesMap = new Map([[0, Buffer.from("truth")]]);
787+
psnrDbValue = 20;
788+
regionShiftError = new Error("No such filter: 'blend'");
789+
const prevBatch = process.env.HF_DE_BATCH;
790+
process.env.HF_DE_BATCH = "1";
791+
const { runCaptureStreamingStage } = await import("./captureStreamingStage.js");
792+
const cfg = { forceScreenshot: false, ffmpegStreamingTimeout: 3_600_000 };
793+
const input = { ...createInput(cfg), totalFrames: 1, workerCount: 1 };
794+
795+
let caught: unknown;
796+
try {
797+
await runCaptureStreamingStage(input);
798+
} catch (error) {
799+
caught = error;
800+
} finally {
801+
sessionWorkerEncodeEnabled = false;
802+
deVerifyFramesMap = undefined;
803+
psnrDbValue = Infinity;
804+
regionShiftError = null;
805+
if (prevBatch === undefined) delete process.env.HF_DE_BATCH;
806+
else process.env.HF_DE_BATCH = prevBatch;
807+
}
808+
809+
expect(caught).toBeInstanceOf(Error);
810+
expect((caught as Error).message).toContain("drawElement self-verify failed at frame 0");
811+
expect((caught as Error).message).toContain("20.0dB < 32dB");
812+
});
813+
814+
it("accepts a verified frame when only the region-shift sample fails", async () => {
815+
// Same isolation: a passing PSNR with a failed shift metric completes
816+
// the render and records no observed max shift.
817+
sessionWorkerEncodeEnabled = true;
818+
deVerifyFramesMap = new Map([[0, Buffer.from("truth")]]);
819+
psnrDbValue = 48;
820+
regionShiftError = new Error("No such filter: 'blend'");
821+
const prevBatch = process.env.HF_DE_BATCH;
822+
process.env.HF_DE_BATCH = "1";
823+
const { runCaptureStreamingStage } = await import("./captureStreamingStage.js");
824+
const cfg = { forceScreenshot: false, ffmpegStreamingTimeout: 3_600_000 };
825+
const input = { ...createInput(cfg), totalFrames: 1, workerCount: 1 };
826+
827+
let result:
828+
| { success?: boolean; deDrainStats?: { verifyMaxShiftObserved?: number } }
829+
| undefined;
830+
let caught: unknown;
831+
try {
832+
result = await runCaptureStreamingStage(input);
833+
} catch (error) {
834+
caught = error;
835+
} finally {
836+
sessionWorkerEncodeEnabled = false;
837+
deVerifyFramesMap = undefined;
838+
psnrDbValue = Infinity;
839+
regionShiftError = null;
840+
if (prevBatch === undefined) delete process.env.HF_DE_BATCH;
841+
else process.env.HF_DE_BATCH = prevBatch;
842+
}
843+
844+
expect(caught).toBeUndefined();
845+
expect(result?.success).toBe(true);
846+
expect(result?.deDrainStats?.verifyMaxShiftObserved).toBeUndefined();
847+
});
775848
});
776849

777850
describe("runCaptureStage", () => {

‎packages/producer/src/services/render/stages/captureStreamingStage.ts‎

Lines changed: 30 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -427,10 +427,20 @@ function createDrainFrameGuard(args: {
427427
}
428428
const truth = session.deVerifyFrames?.get(idx);
429429
if (truth) {
430+
// The shift metric is observe-only and its ffmpeg chain uses `blend`,
431+
// which the bootstrap preflight doesn't cover — run it under its own
432+
// catch so a shift failure can't forfeit the enforced PSNR result too.
433+
// Log and move on: the PSNR gate stays exactly as strong as on main.
434+
const shiftP = regionShift(buf, truth).catch((err: unknown) => {
435+
log.warn("[Render] drawElement region-shift sample failed; metric skipped", {
436+
frame: idx,
437+
error: err instanceof Error ? err.message : String(err),
438+
});
439+
return undefined;
440+
});
430441
let db: number;
431-
let shift: number;
432442
try {
433-
[db, shift] = await Promise.all([psnrDb(buf, truth), regionShift(buf, truth)]);
443+
db = await psnrDb(buf, truth);
434444
} catch (err) {
435445
// Infrastructure failure (ffmpeg spawn/parse/tmpdir), not evidence of
436446
// damage — skip this sample rather than failing or falling back.
@@ -440,7 +450,11 @@ function createDrainFrameGuard(args: {
440450
});
441451
return buf;
442452
}
443-
if (db < verifyMinDb || (verifyMaxShift !== undefined && shift > verifyMaxShift)) {
453+
const shift = await shiftP;
454+
if (
455+
db < verifyMinDb ||
456+
(shift !== undefined && verifyMaxShift !== undefined && shift > verifyMaxShift)
457+
) {
444458
// Keep the mismatched pair for diagnosis (tmpdir; OS-reaped).
445459
const dumpDir = await mkdtemp(join(tmpdir(), "hf-de-verify-fail-")).catch(() => null);
446460
if (dumpDir) {
@@ -455,21 +469,25 @@ function createDrainFrameGuard(args: {
455469
{ kind: "psnr", frameIndex: idx, failedDb: db, verifyThresholdDb: verifyMinDb },
456470
);
457471
}
458-
throw new DrawElementVerificationError(
459-
`drawElement self-verify failed at frame ${idx}: region shift ${shift}/255 > ${verifyMaxShift}/255 vs pre-injection screenshot; a low-contrast region is absent from the drawElement frame${dumpDir ? ` (pair: ${dumpDir})` : ""}`,
460-
{ kind: "shift", frameIndex: idx, failedShift: shift, verifyMaxShift },
461-
);
472+
if (shift !== undefined && verifyMaxShift !== undefined) {
473+
throw new DrawElementVerificationError(
474+
`drawElement self-verify failed at frame ${idx}: region shift ${shift}/255 > ${verifyMaxShift}/255 vs pre-injection screenshot; a low-contrast region is absent from the drawElement frame${dumpDir ? ` (pair: ${dumpDir})` : ""}`,
475+
{ kind: "shift", frameIndex: idx, failedShift: shift, verifyMaxShift },
476+
);
477+
}
462478
}
463479
stats.verifyChecked += 1;
464480
stats.verifyMinDb = stats.verifyMinDb === undefined ? db : Math.min(stats.verifyMinDb, db);
465-
stats.verifyMaxShiftObserved =
466-
stats.verifyMaxShiftObserved === undefined
467-
? shift
468-
: Math.max(stats.verifyMaxShiftObserved, shift);
481+
if (shift !== undefined) {
482+
stats.verifyMaxShiftObserved =
483+
stats.verifyMaxShiftObserved === undefined
484+
? shift
485+
: Math.max(stats.verifyMaxShiftObserved, shift);
486+
}
469487
log.info("[Render] drawElement self-verify passed", {
470488
frame: idx,
471489
psnrDb: db === Infinity ? "inf" : Number(db.toFixed(1)),
472-
regionShift: shift,
490+
regionShift: shift === undefined ? "skipped" : shift,
473491
});
474492
}
475493
return buf;

0 commit comments

Comments
 (0)