Skip to content

Commit c7092fa

Browse files
TianT1209TianT1209
committed
fix: decode BOM-prefixed Windows exec output
1 parent 56d0721 commit c7092fa

2 files changed

Lines changed: 179 additions & 32 deletions

File tree

‎src/infra/windows-encoding.test.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,46 @@ describe("windows output encoding", () => {
108108
).toBe("测试");
109109
});
110110

111+
it("decodes UTF-16LE BOM-prefixed Windows output as text", () => {
112+
const raw = Buffer.from([
113+
0xff, 0xfe, 0x68, 0x00, 0x65, 0x00, 0x6c, 0x00, 0x6c, 0x00, 0x6f, 0x00, 0x0a, 0x00,
114+
]);
115+
116+
expect(
117+
decodeWindowsOutputBuffer({
118+
buffer: raw,
119+
platform: "win32",
120+
windowsEncoding: "utf-8",
121+
}),
122+
).toBe("hello\n");
123+
});
124+
125+
it("decodes UTF-8 BOM-prefixed Windows output as text", () => {
126+
const raw = Buffer.from([0xef, 0xbb, 0xbf, ...Buffer.from("hello\n", "utf8")]);
127+
128+
expect(
129+
decodeWindowsOutputBuffer({
130+
buffer: raw,
131+
platform: "win32",
132+
windowsEncoding: "gbk",
133+
}),
134+
).toBe("hello\n");
135+
});
136+
137+
it("decodes UTF-16BE BOM-prefixed Windows output as text", () => {
138+
const raw = Buffer.from([
139+
0xfe, 0xff, 0x00, 0x68, 0x00, 0x65, 0x00, 0x6c, 0x00, 0x6c, 0x00, 0x6f, 0x00, 0x0a,
140+
]);
141+
142+
expect(
143+
decodeWindowsOutputBuffer({
144+
buffer: raw,
145+
platform: "win32",
146+
windowsEncoding: "utf-8",
147+
}),
148+
).toBe("hello\n");
149+
});
150+
111151
it("decodes legacy text files with the Windows system encoding", () => {
112152
const raw = Buffer.from([0xc4, 0xe3, 0xba, 0xc3]);
113153

@@ -186,4 +226,19 @@ describe("windows output encoding", () => {
186226
expect(decoder.decode(raw.subarray(splitIndex + 1))).toBe('世"}');
187227
expect(decoder.flush()).toBe("");
188228
});
229+
230+
it("decodes split UTF-16LE BOM-prefixed Windows output chunks as text", () => {
231+
const decoder = createWindowsOutputDecoder({
232+
platform: "win32",
233+
windowsEncoding: "utf-8",
234+
});
235+
const raw = Buffer.from([
236+
0xff, 0xfe, 0x68, 0x00, 0x65, 0x00, 0x6c, 0x00, 0x6c, 0x00, 0x6f, 0x00, 0x0a, 0x00,
237+
]);
238+
239+
expect(decoder.decode(raw.subarray(0, 1))).toBe("");
240+
expect(decoder.decode(raw.subarray(1, 5))).toBe("h");
241+
expect(decoder.decode(raw.subarray(5))).toBe("ello\n");
242+
expect(decoder.flush()).toBe("");
243+
});
189244
});

‎src/infra/windows-encoding.ts‎

Lines changed: 124 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,11 @@ function decodeWindowsBufferWithFallback(params: {
132132
return params.buffer.toString("utf8");
133133
}
134134

135+
const bomDecoded = decodeBomPrefixedBuffer(params.buffer);
136+
if (bomDecoded !== null) {
137+
return bomDecoded;
138+
}
139+
135140
const utf8 = decodeStrictUtf8(params.buffer);
136141
if (utf8 !== null) {
137142
return utf8;
@@ -148,6 +153,49 @@ function decodeWindowsBufferWithFallback(params: {
148153
}
149154
}
150155

156+
type TextBomState =
157+
| { status: "matched"; encoding: "utf-8" | "utf-16le" | "utf-16be" }
158+
| { status: "pending" }
159+
| { status: "none" };
160+
161+
function detectTextBom(buffer: Buffer): TextBomState {
162+
const first = buffer.at(0);
163+
if (first === undefined) {
164+
return { status: "none" };
165+
}
166+
const second = buffer.at(1);
167+
const third = buffer.at(2);
168+
if (first === 0xff) {
169+
if (second === undefined) {
170+
return { status: "pending" };
171+
}
172+
return second === 0xfe ? { status: "matched", encoding: "utf-16le" } : { status: "none" };
173+
}
174+
if (first === 0xfe) {
175+
if (second === undefined) {
176+
return { status: "pending" };
177+
}
178+
return second === 0xff ? { status: "matched", encoding: "utf-16be" } : { status: "none" };
179+
}
180+
if (first === 0xef) {
181+
if (second === undefined || (second === 0xbb && third === undefined)) {
182+
return { status: "pending" };
183+
}
184+
return second === 0xbb && third === 0xbf
185+
? { status: "matched", encoding: "utf-8" }
186+
: { status: "none" };
187+
}
188+
return { status: "none" };
189+
}
190+
191+
function decodeBomPrefixedBuffer(buffer: Buffer): string | null {
192+
const bom = detectTextBom(buffer);
193+
if (bom.status !== "matched") {
194+
return null;
195+
}
196+
return new TextDecoder(bom.encoding).decode(buffer);
197+
}
198+
151199
/** Creates a streaming decoder for subprocess output chunks that may split multibyte characters. */
152200
export function createWindowsOutputDecoder(params?: {
153201
platform?: NodeJS.Platform;
@@ -168,48 +216,92 @@ export function createWindowsOutputDecoder(params?: {
168216
platform === "win32" && legacyDecoder ? new TextDecoder("utf-8", { fatal: true }) : null;
169217
const streamingUtf8Decoder = legacyDecoder ? null : new TextDecoder("utf-8");
170218
let useLegacyDecoder = false;
171-
let pendingUtf8Bytes = Buffer.alloc(0);
219+
let pendingUtf8Bytes: Buffer = Buffer.alloc(0);
220+
let bomChecked = platform !== "win32";
221+
let pendingBomBytes: Buffer = Buffer.alloc(0);
222+
let bomDecoder: TextDecoder | null = null;
223+
224+
const decodeWithoutBom = (buffer: Buffer) => {
225+
if (!legacyDecoder || !utf8Decoder) {
226+
return streamingUtf8Decoder?.decode(buffer, { stream: true }) ?? "";
227+
}
228+
if (useLegacyDecoder) {
229+
return legacyDecoder.decode(buffer, { stream: true });
230+
}
231+
// Stay on strict UTF-8 until it fails; replay any pending lead bytes through the legacy
232+
// decoder so split GBK/Big5/etc. characters are not lost at the fallback boundary.
233+
const replayBuffer =
234+
pendingUtf8Bytes.length > 0 ? Buffer.concat([pendingUtf8Bytes, buffer]) : buffer;
235+
try {
236+
const decoded = utf8Decoder.decode(buffer, { stream: true });
237+
pendingUtf8Bytes = Buffer.from(getTrailingIncompleteUtf8Bytes(replayBuffer));
238+
return decoded;
239+
} catch {
240+
useLegacyDecoder = true;
241+
pendingUtf8Bytes = Buffer.alloc(0);
242+
return legacyDecoder.decode(replayBuffer, { stream: true });
243+
}
244+
};
245+
246+
const flushWithoutBom = () => {
247+
if (!legacyDecoder || !utf8Decoder) {
248+
return streamingUtf8Decoder?.decode() ?? "";
249+
}
250+
if (useLegacyDecoder) {
251+
return legacyDecoder.decode();
252+
}
253+
try {
254+
const decoded = utf8Decoder.decode();
255+
pendingUtf8Bytes = Buffer.alloc(0);
256+
return decoded;
257+
} catch {
258+
useLegacyDecoder = true;
259+
const replayBuffer = pendingUtf8Bytes;
260+
pendingUtf8Bytes = Buffer.alloc(0);
261+
return replayBuffer.length > 0 ? legacyDecoder.decode(replayBuffer) : "";
262+
}
263+
};
172264

173265
return {
174266
decode(chunk) {
175267
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
176-
if (!legacyDecoder || !utf8Decoder) {
177-
return streamingUtf8Decoder?.decode(buffer, { stream: true }) ?? "";
178-
}
179-
if (useLegacyDecoder) {
180-
return legacyDecoder.decode(buffer, { stream: true });
268+
if (bomDecoder) {
269+
return bomDecoder.decode(buffer, { stream: true });
181270
}
182-
// Stay on strict UTF-8 until it fails; replay any pending lead bytes through the legacy
183-
// decoder so split GBK/Big5/etc. characters are not lost at the fallback boundary.
184-
const replayBuffer =
185-
pendingUtf8Bytes.length > 0 ? Buffer.concat([pendingUtf8Bytes, buffer]) : buffer;
186-
try {
187-
const decoded = utf8Decoder.decode(buffer, { stream: true });
188-
pendingUtf8Bytes = Buffer.from(getTrailingIncompleteUtf8Bytes(replayBuffer));
189-
return decoded;
190-
} catch {
191-
useLegacyDecoder = true;
192-
pendingUtf8Bytes = Buffer.alloc(0);
193-
return legacyDecoder.decode(replayBuffer, { stream: true });
271+
if (!bomChecked) {
272+
const candidate =
273+
pendingBomBytes.length > 0 ? Buffer.concat([pendingBomBytes, buffer]) : buffer;
274+
const bom = detectTextBom(candidate);
275+
if (bom.status === "pending") {
276+
pendingBomBytes = candidate;
277+
return "";
278+
}
279+
bomChecked = true;
280+
pendingBomBytes = Buffer.alloc(0);
281+
if (bom.status === "matched") {
282+
bomDecoder = new TextDecoder(bom.encoding);
283+
return bomDecoder.decode(candidate, { stream: true });
284+
}
285+
return decodeWithoutBom(candidate);
194286
}
287+
return decodeWithoutBom(buffer);
195288
},
196289
flush() {
197-
if (!legacyDecoder || !utf8Decoder) {
198-
return streamingUtf8Decoder?.decode() ?? "";
199-
}
200-
if (useLegacyDecoder) {
201-
return legacyDecoder.decode();
290+
if (bomDecoder) {
291+
return bomDecoder.decode();
202292
}
203-
try {
204-
const decoded = utf8Decoder.decode();
205-
pendingUtf8Bytes = Buffer.alloc(0);
206-
return decoded;
207-
} catch {
208-
useLegacyDecoder = true;
209-
const replayBuffer = pendingUtf8Bytes;
210-
pendingUtf8Bytes = Buffer.alloc(0);
211-
return replayBuffer.length > 0 ? legacyDecoder.decode(replayBuffer) : "";
293+
if (!bomChecked && pendingBomBytes.length > 0) {
294+
bomChecked = true;
295+
const pending = pendingBomBytes;
296+
pendingBomBytes = Buffer.alloc(0);
297+
const bom = detectTextBom(pending);
298+
if (bom.status === "matched") {
299+
bomDecoder = new TextDecoder(bom.encoding);
300+
return bomDecoder.decode(pending);
301+
}
302+
return decodeWithoutBom(pending) + flushWithoutBom();
212303
}
304+
return flushWithoutBom();
213305
},
214306
};
215307
}

0 commit comments

Comments
 (0)