Skip to content

Commit a707b6a

Browse files
jrusso1020claude
andcommitted
fix(cli): pin inverse-alpha invariants, harden encoder stdin
- Extract applyMask helper from postprocess and add 5 unit tests pinning the contract this PR is selling: fg.alpha + bg.alpha === 255 per pixel, RGB triples byte-identical between fg and bg, and bg=null path leaves the bg buffer untouched. Without these, a future postprocess change (mask threshold, premultiplied alpha, gamma) could silently break the inverse-alpha relationship and the existing plumbing tests would all still pass. - Add stdin 'error' listener inside spawnFfmpeg. If either encoder dies mid-render, Node emits an unhandled error on the dead writable on the next .write() and crashes the CLI before waitForExit's reject path can surface the encoder's stderr tail. Doubled encoder count = doubled failure surface, so this is worth pinning down. - Tighten stdio param to a 3-tuple so an accidental 1-element array fails at type-check. - Sharpen backpressure comment: write→true means "highWaterMark not exceeded," not "libuv flushed." Reuse-without-corruption is safe only because session.process is slow enough that libuv drains in between. Addresses review on PR #637. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
1 parent c2bc2aa commit a707b6a

3 files changed

Lines changed: 164 additions & 19 deletions

File tree

‎packages/cli/src/background-removal/inference.test.ts‎

Lines changed: 110 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { describe, expect, it } from "vitest";
2-
import { MEAN, STD } from "./inference.js";
2+
import { MEAN, STD, applyMask } from "./inference.js";
33

44
// Regression: the u2net_human_seg model was trained with ImageNet
55
// normalization. Drifting away from these exact values changes the input
@@ -16,3 +16,112 @@ describe("background-removal/inference — rembg u2net_human_seg parity", () =>
1616
expect(STD).toEqual([0.229, 0.224, 0.225]);
1717
});
1818
});
19+
20+
// These tests pin the contract that `--background-output` is built on:
21+
// fg.alpha + bg.alpha === 255 per pixel, and the RGB plane is byte-identical
22+
// between fg and bg. A future change to the postprocess loop (different mask
23+
// threshold, premultiplied alpha, gamma-corrected compositing) that breaks
24+
// either invariant should fail here loudly.
25+
describe("background-removal/inference — applyMask invariants", () => {
26+
function makeRgb(pixels: number): Buffer {
27+
// Deterministic but non-trivial RGB so byte equality is meaningful.
28+
const buf = Buffer.allocUnsafe(pixels * 3);
29+
for (let i = 0; i < pixels; i++) {
30+
buf[i * 3] = (i * 7) & 0xff;
31+
buf[i * 3 + 1] = (i * 13 + 31) & 0xff;
32+
buf[i * 3 + 2] = (i * 19 + 61) & 0xff;
33+
}
34+
return buf;
35+
}
36+
37+
function makeMask(pixels: number): Buffer {
38+
// Hit the saturation endpoints (0, 255) and a few mid-tone values so the
39+
// 255-m inversion is exercised across the full byte range.
40+
const buf = Buffer.allocUnsafe(pixels);
41+
for (let i = 0; i < pixels; i++) buf[i] = (i * 37) & 0xff;
42+
return buf;
43+
}
44+
45+
it("dual-output: fg.alpha + bg.alpha === 255 for every pixel", () => {
46+
const pixels = 64;
47+
const rgb = makeRgb(pixels);
48+
const mask = makeMask(pixels);
49+
const fg = Buffer.allocUnsafe(pixels * 4);
50+
const bg = Buffer.allocUnsafe(pixels * 4);
51+
52+
const result = applyMask(rgb, mask, fg, bg, pixels);
53+
54+
expect(result.fg).toBe(fg);
55+
expect(result.bg).toBe(bg);
56+
for (let i = 0; i < pixels; i++) {
57+
const sum = fg[i * 4 + 3]! + bg[i * 4 + 3]!;
58+
expect(sum).toBe(255);
59+
}
60+
});
61+
62+
it("dual-output: RGB triples are byte-identical between fg and bg", () => {
63+
const pixels = 64;
64+
const rgb = makeRgb(pixels);
65+
const mask = makeMask(pixels);
66+
const fg = Buffer.allocUnsafe(pixels * 4);
67+
const bg = Buffer.allocUnsafe(pixels * 4);
68+
69+
applyMask(rgb, mask, fg, bg, pixels);
70+
71+
for (let i = 0; i < pixels; i++) {
72+
expect(fg[i * 4]).toBe(bg[i * 4]);
73+
expect(fg[i * 4 + 1]).toBe(bg[i * 4 + 1]);
74+
expect(fg[i * 4 + 2]).toBe(bg[i * 4 + 2]);
75+
// And both match the source.
76+
expect(fg[i * 4]).toBe(rgb[i * 3]);
77+
expect(fg[i * 4 + 1]).toBe(rgb[i * 3 + 1]);
78+
expect(fg[i * 4 + 2]).toBe(rgb[i * 3 + 2]);
79+
}
80+
});
81+
82+
it("dual-output: fg.alpha equals the input mask", () => {
83+
const pixels = 32;
84+
const rgb = makeRgb(pixels);
85+
const mask = makeMask(pixels);
86+
const fg = Buffer.allocUnsafe(pixels * 4);
87+
const bg = Buffer.allocUnsafe(pixels * 4);
88+
89+
applyMask(rgb, mask, fg, bg, pixels);
90+
91+
for (let i = 0; i < pixels; i++) {
92+
expect(fg[i * 4 + 3]).toBe(mask[i]);
93+
}
94+
});
95+
96+
it("single-output: bg=null returns bg=null and writes only fg", () => {
97+
const pixels = 32;
98+
const rgb = makeRgb(pixels);
99+
const mask = makeMask(pixels);
100+
const fg = Buffer.allocUnsafe(pixels * 4);
101+
102+
const result = applyMask(rgb, mask, fg, null, pixels);
103+
104+
expect(result.bg).toBeNull();
105+
expect(result.fg).toBe(fg);
106+
for (let i = 0; i < pixels; i++) {
107+
expect(fg[i * 4]).toBe(rgb[i * 3]);
108+
expect(fg[i * 4 + 3]).toBe(mask[i]);
109+
}
110+
});
111+
112+
it("saturates correctly at mask=0 and mask=255", () => {
113+
// mask=0 → fg.alpha=0 (transparent subject), bg.alpha=255 (fully opaque plate)
114+
// mask=255 → fg.alpha=255 (fully opaque subject), bg.alpha=0 (transparent plate)
115+
const rgb = Buffer.from([10, 20, 30, 40, 50, 60]);
116+
const mask = Buffer.from([0, 255]);
117+
const fg = Buffer.allocUnsafe(8);
118+
const bg = Buffer.allocUnsafe(8);
119+
120+
applyMask(rgb, mask, fg, bg, 2);
121+
122+
expect(fg[3]).toBe(0);
123+
expect(bg[3]).toBe(255);
124+
expect(fg[7]).toBe(255);
125+
expect(bg[7]).toBe(0);
126+
});
127+
});

‎packages/cli/src/background-removal/inference.ts‎

Lines changed: 37 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -200,30 +200,50 @@ async function postprocess(
200200
.raw()
201201
.toBuffer();
202202

203-
const pixels = width * height;
204-
if (rgbaBgBuf) {
203+
return applyMask(rgb, fullMask, rgbaBuf, rgbaBgBuf, width * height);
204+
}
205+
206+
/**
207+
* Composite the RGB source frame with the segmentation mask into one or two
208+
* RGBA buffers. The contract this PR is built on:
209+
* - `fg`'s alpha is the mask, `bg`'s alpha (when provided) is `255 − mask`,
210+
* so `fg.alpha + bg.alpha === 255` for every pixel.
211+
* - RGB triples are byte-identical between `fg` and `bg`.
212+
* - When `bg` is null, only `fg` is touched.
213+
*
214+
* Exported for direct unit testing of the invariants above without spinning
215+
* up an ONNX session.
216+
*/
217+
export function applyMask(
218+
rgb: Buffer,
219+
mask: Buffer,
220+
fg: Buffer,
221+
bg: Buffer | null,
222+
pixels: number,
223+
): SessionResult {
224+
if (bg) {
205225
for (let i = 0; i < pixels; i++) {
206226
const r = rgb[i * 3]!;
207227
const g = rgb[i * 3 + 1]!;
208228
const b = rgb[i * 3 + 2]!;
209-
const m = fullMask[i]!;
229+
const m = mask[i]!;
210230
const o = i * 4;
211-
rgbaBuf[o] = r;
212-
rgbaBuf[o + 1] = g;
213-
rgbaBuf[o + 2] = b;
214-
rgbaBuf[o + 3] = m;
215-
rgbaBgBuf[o] = r;
216-
rgbaBgBuf[o + 1] = g;
217-
rgbaBgBuf[o + 2] = b;
218-
rgbaBgBuf[o + 3] = 255 - m;
231+
fg[o] = r;
232+
fg[o + 1] = g;
233+
fg[o + 2] = b;
234+
fg[o + 3] = m;
235+
bg[o] = r;
236+
bg[o + 1] = g;
237+
bg[o + 2] = b;
238+
bg[o + 3] = 255 - m;
219239
}
220-
return { fg: rgbaBuf, bg: rgbaBgBuf };
240+
return { fg, bg };
221241
}
222242
for (let i = 0; i < pixels; i++) {
223-
rgbaBuf[i * 4] = rgb[i * 3]!;
224-
rgbaBuf[i * 4 + 1] = rgb[i * 3 + 1]!;
225-
rgbaBuf[i * 4 + 2] = rgb[i * 3 + 2]!;
226-
rgbaBuf[i * 4 + 3] = fullMask[i]!;
243+
fg[i * 4] = rgb[i * 3]!;
244+
fg[i * 4 + 1] = rgb[i * 3 + 1]!;
245+
fg[i * 4 + 2] = rgb[i * 3 + 2]!;
246+
fg[i * 4 + 3] = mask[i]!;
227247
}
228-
return { fg: rgbaBuf, bg: null };
248+
return { fg, bg: null };
229249
}

‎packages/cli/src/background-removal/pipeline.ts‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -318,12 +318,21 @@ interface FfmpegProc {
318318
getStderr: () => string;
319319
}
320320

321-
function spawnFfmpeg(args: string[], label: string, stdio: ("ignore" | "pipe")[]): FfmpegProc {
321+
type StdioFd = "ignore" | "pipe";
322+
type StdioTuple = [StdioFd, StdioFd, StdioFd];
323+
324+
function spawnFfmpeg(args: string[], label: string, stdio: StdioTuple): FfmpegProc {
322325
const proc = spawn("ffmpeg", args, { stdio });
323326
let stderrBuf = "";
324327
proc.stderr?.on("data", (d: Buffer) => {
325328
stderrBuf += d.toString();
326329
});
330+
// If the encoder dies mid-render, the next .write() to its stdin emits an
331+
// 'error' event on the writable. Without a listener, Node treats it as
332+
// unhandled and crashes the CLI before waitForExit's reject path can
333+
// surface the real cause (encoder stderr tail). Swallowing here is safe —
334+
// the process exit is the source of truth.
335+
proc.stdin?.on("error", () => {});
327336
const exit = waitForExit(proc, label, () => stderrBuf);
328337
return { proc, exit, getStderr: () => stderrBuf };
329338
}
@@ -383,6 +392,13 @@ async function runPipeline(
383392
// Issue both writes before any await so a slow encoder doesn't block
384393
// the other. Drain anything that returned false before the next
385394
// session.process() — its output buffers are reused per frame.
395+
//
396+
// Subtlety: write() returning true means "highWaterMark not exceeded,"
397+
// NOT "libuv has flushed the chunk." The buffer reference is held by
398+
// libuv until the underlying syscall completes. Reusing the session's
399+
// output buffer is safe because the next session.process() call takes
400+
// ~10–50ms (ORT inference) — plenty of event-loop turns for libuv to
401+
// drain. If that ever stops being true, we'd need to copy here.
386402
const fgWroteFully = fg.proc.stdin!.write(result.fg);
387403
const bgWroteFully = bg && result.bg ? bg.proc.stdin!.write(result.bg) : true;
388404
if (!fgWroteFully || !bgWroteFully) {

0 commit comments

Comments
 (0)