Skip to content

Commit 7246680

Browse files
committed
fix(server): decode Windows CLI output written in the ANSI code page
Python CLIs such as az are not in UTF-8 mode, so their piped stdout is encoded in the system ANSI code page. Every non-ASCII character in Azure DevOps PR titles, descriptions and comments rendered as U+FFFD because process output was decoded as UTF-8 only. Decode a buffer that is not valid UTF-8 with the ANSI code page the host reports, and keep the replacement character for a tail cut by truncation. The code page query runs %SystemRoot%\System32\reg.exe rather than a bare reg, so the current directory cannot supply the executable.
1 parent 0b02b21 commit 7246680

7 files changed

Lines changed: 247 additions & 7 deletions

File tree

‎apps/server/src/processRunner.test.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import * as Stream from "effect/Stream";
88
import { ChildProcessSpawner } from "effect/process";
99
import * as HostProcess from "@t3tools/shared/HostProcess";
1010
import { SpawnExecutableResolution } from "@t3tools/shared/shell";
11+
import { WindowsAnsiEncoding } from "@t3tools/shared/windowsCodePage";
1112

1213
import * as ProcessRunner from "./processRunner.ts";
1314

@@ -231,6 +232,41 @@ describe("runProcess", () => {
231232
}),
232233
);
233234

235+
it.effect("decodes Windows CLI output written in the ANSI code page", () => {
236+
const spawner = makeSpawner(() =>
237+
Effect.succeed(makeHandle({ stdout: Stream.make(new Uint8Array([0x63, 0x61, 0x66, 0xe9])) })),
238+
);
239+
240+
return runWith(spawner)({
241+
command: "az",
242+
args: ["repos", "pr", "show"],
243+
}).pipe(
244+
Effect.provideService(HostProcess.Platform, "win32"),
245+
Effect.provideService(WindowsAnsiEncoding, Effect.succeed("windows-1252")),
246+
Effect.map((result) => {
247+
expect(result.stdout).toBe("caf\u00e9");
248+
expect(result.stdoutInvalidUtf8).toBe(true);
249+
}),
250+
);
251+
});
252+
253+
it.effect("keeps invalid UTF-8 outside Windows", () => {
254+
const spawner = makeSpawner(() =>
255+
Effect.succeed(makeHandle({ stdout: Stream.make(new Uint8Array([0x63, 0x61, 0x66, 0xe9])) })),
256+
);
257+
258+
return runWith(spawner)({
259+
command: "az",
260+
args: ["repos", "pr", "show"],
261+
}).pipe(
262+
Effect.provideService(HostProcess.Platform, "linux"),
263+
Effect.provideService(WindowsAnsiEncoding, Effect.succeed("windows-1252")),
264+
Effect.map((result) => {
265+
expect(result.stdout).toBe("caf\uFFFD");
266+
}),
267+
);
268+
});
269+
234270
it.effect("writes stdin before waiting for exit", () =>
235271
Effect.gen(function* () {
236272
const stdinWritten = yield* Deferred.make<void>();

‎apps/server/src/processRunner.ts‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@ import * as Scope from "effect/Scope";
99
import * as Stream from "effect/Stream";
1010
import * as ChildProcess from "effect/process/ChildProcess";
1111
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
12+
import * as HostProcess from "@t3tools/shared/HostProcess";
1213
import { resolveSpawnCommand } from "@t3tools/shared/shell";
14+
import { WindowsAnsiEncoding } from "@t3tools/shared/windowsCodePage";
1315
import {
1416
collectUint8StreamText,
1517
decodeUtf8,
@@ -159,6 +161,7 @@ const collectText = Effect.fnUntraced(function* (input: {
159161
readonly maxOutputBytes: number;
160162
readonly outputMode: "error" | "truncate";
161163
readonly truncatedMarker: string;
164+
readonly legacyEncoding: string | null;
162165
}) {
163166
const stream = input.stream.pipe(
164167
Stream.mapError(
@@ -179,6 +182,7 @@ const collectText = Effect.fnUntraced(function* (input: {
179182
stream,
180183
maxBytes: input.maxOutputBytes,
181184
truncatedMarker: input.truncatedMarker,
185+
legacyEncoding: input.legacyEncoding,
182186
});
183187
}
184188

@@ -217,7 +221,9 @@ const collectText = Effect.fnUntraced(function* (input: {
217221
},
218222
),
219223
Effect.map((state): CollectedUint8StreamText => ({
220-
...decodeUtf8(Buffer.concat(state.chunks, state.bytes)),
224+
...decodeUtf8(Buffer.concat(state.chunks, state.bytes), {
225+
legacyEncoding: input.legacyEncoding,
226+
}),
221227
bytes: state.bytes,
222228
truncated: false,
223229
})),
@@ -267,12 +273,14 @@ const commandName = (command: string) => command.replace(/^.*[\\/]/, "");
267273

268274
const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
269275
spawner: ChildProcessSpawner.ChildProcessSpawner["Service"],
276+
legacyOutputEncoding: Effect.Effect<string | null>,
270277
input: ProcessRunInput,
271278
): Effect.fn.Return<ProcessRunOutput, ProcessRunError, Scope.Scope> {
272279
yield* Effect.annotateCurrentSpan("process.command", commandName(input.command));
273280
const maxOutputBytes = input.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
274281
const outputMode = input.outputMode ?? "error";
275282
const truncatedMarker = input.truncatedMarker ?? "";
283+
const legacyEncoding = yield* legacyOutputEncoding;
276284
const extendEnv = input.env !== undefined;
277285
const spawnCommand = yield* resolveSpawnCommand(
278286
input.command,
@@ -345,6 +353,7 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
345353
maxOutputBytes,
346354
outputMode,
347355
truncatedMarker,
356+
legacyEncoding,
348357
}),
349358
collectText({
350359
command: input.command,
@@ -356,6 +365,7 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
356365
maxOutputBytes,
357366
outputMode,
358367
truncatedMarker,
368+
legacyEncoding,
359369
}),
360370
writeStdin,
361371
],
@@ -391,9 +401,14 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
391401
/** @public */
392402
export const make = Effect.fn("ProcessRunner.make")(function* () {
393403
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
404+
const platform = yield* HostProcess.Platform;
405+
// Python CLIs (`az`, `aws`) that are not in UTF-8 mode encode piped output in the ANSI code page.
406+
const legacyEncoding: Effect.Effect<string | null> =
407+
platform === "win32" ? yield* WindowsAnsiEncoding : Effect.succeed(null);
408+
const legacyOutputEncoding = yield* Effect.cached(legacyEncoding);
394409

395410
const run: ProcessRunner["Service"]["run"] = (input) =>
396-
finalizeRunProcess(runProcessCore(spawner, input), input);
411+
finalizeRunProcess(runProcessCore(spawner, legacyOutputEncoding, input), input);
397412

398413
return ProcessRunner.of({
399414
run,

‎packages/provider-core/src/server/collectStreamText.test.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,4 +39,30 @@ describe("collectUint8StreamText", () => {
3939
assert.strictEqual(literal.text, "before\uFFFDafter");
4040
}),
4141
);
42+
43+
it.effect("decodes ANSI-encoded output with the given legacy encoding", () =>
44+
Effect.gen(function* () {
45+
const result = yield* collectUint8StreamText({
46+
stream: Stream.make(new Uint8Array([0x63, 0x61, 0x66, 0xe9])),
47+
legacyEncoding: "windows-1252",
48+
});
49+
50+
assert.strictEqual(result.text, "caf\u00e9");
51+
assert.strictEqual(result.invalidUtf8, true);
52+
}),
53+
);
54+
55+
it.effect("keeps a truncated multi-byte character a replacement character", () =>
56+
Effect.gen(function* () {
57+
const result = yield* collectUint8StreamText({
58+
stream: Stream.make(encoder.encode("caf\u00e9"), encoder.encode(" next")),
59+
maxBytes: 4,
60+
truncatedMarker: "[truncated]",
61+
legacyEncoding: "windows-1252",
62+
});
63+
64+
assert.strictEqual(result.text, "caf\uFFFD[truncated]");
65+
assert.strictEqual(result.invalidUtf8, true);
66+
}),
67+
);
4268
});

‎packages/provider-core/src/server/collectStreamText.ts‎

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,36 @@ export interface CollectedUint8StreamText {
99
readonly invalidUtf8: boolean;
1010
}
1111

12+
export interface DecodeStreamTextOptions {
13+
/** Decodes output that is not UTF-8, such as a Windows CLI writing its ANSI code page. */
14+
readonly legacyEncoding?: string | null | undefined;
15+
/** Truncation can cut a multi-byte character, so a short tail alone is no evidence of another encoding. */
16+
readonly truncated?: boolean | undefined;
17+
}
18+
19+
const decodeWithEncoding = (bytes: Uint8Array, encoding: string): string => {
20+
try {
21+
return new TextDecoder(encoding).decode(bytes);
22+
} catch {
23+
return Buffer.from(bytes).toString("utf8");
24+
}
25+
};
26+
1227
export const decodeUtf8 = (
1328
bytes: Uint8Array,
14-
): Pick<CollectedUint8StreamText, "text" | "invalidUtf8"> => ({
15-
text: Buffer.from(bytes).toString("utf8"),
16-
invalidUtf8: !NodeBuffer.isUtf8(bytes),
17-
});
29+
options: DecodeStreamTextOptions = {},
30+
): Pick<CollectedUint8StreamText, "text" | "invalidUtf8"> => {
31+
const invalidUtf8 = !NodeBuffer.isUtf8(bytes);
32+
const legacyEncoding = options.legacyEncoding ?? null;
33+
34+
return {
35+
text:
36+
invalidUtf8 && legacyEncoding !== null && options.truncated !== true
37+
? decodeWithEncoding(bytes, legacyEncoding)
38+
: Buffer.from(bytes).toString("utf8"),
39+
invalidUtf8,
40+
};
41+
};
1842

1943
interface CollectState {
2044
chunks: Uint8Array[];
@@ -26,6 +50,7 @@ export const collectUint8StreamText = <E>(input: {
2650
readonly stream: Stream.Stream<Uint8Array, E>;
2751
readonly maxBytes?: number | undefined;
2852
readonly truncatedMarker?: string | null | undefined;
53+
readonly legacyEncoding?: string | null | undefined;
2954
}): Effect.Effect<CollectedUint8StreamText, E> => {
3055
const maxBytes = input.maxBytes ?? Number.POSITIVE_INFINITY;
3156
const truncatedMarker = input.truncatedMarker ?? "";
@@ -65,7 +90,10 @@ export const collectUint8StreamText = <E>(input: {
6590
},
6691
),
6792
Effect.map((state): CollectedUint8StreamText => {
68-
const decoded = decodeUtf8(Buffer.concat(state.chunks, state.bytes));
93+
const decoded = decodeUtf8(Buffer.concat(state.chunks, state.bytes), {
94+
legacyEncoding: input.legacyEncoding,
95+
truncated: state.truncated,
96+
});
6997
return {
7098
text:
7199
state.truncated && truncatedMarker.length > 0

‎packages/shared/package.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -502,6 +502,10 @@
502502
"./cloneUrl": {
503503
"types": "./src/cloneUrl.ts",
504504
"import": "./src/cloneUrl.ts"
505+
},
506+
"./windowsCodePage": {
507+
"types": "./src/windowsCodePage.ts",
508+
"import": "./src/windowsCodePage.ts"
505509
}
506510
},
507511
"scripts": {
Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
import { describe, expect, it } from "vite-plus/test";
2+
3+
import {
4+
ansiEncodingForCodePage,
5+
parseCodePageRegistryOutput,
6+
windowsRegistryTool,
7+
} from "./windowsCodePage.ts";
8+
9+
describe("parseCodePageRegistryOutput", () => {
10+
it("reads the ACP value from `reg query` output", () => {
11+
expect(
12+
parseCodePageRegistryOutput(
13+
[
14+
"",
15+
"HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Nls\\CodePage",
16+
" ACP REG_SZ 1252",
17+
"",
18+
].join("\r\n"),
19+
),
20+
).toBe(1252);
21+
});
22+
23+
it("returns null when the value or the query is missing", () => {
24+
expect(parseCodePageRegistryOutput("")).toBeNull();
25+
expect(
26+
parseCodePageRegistryOutput(
27+
"HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Nls\\CodePage",
28+
),
29+
).toBeNull();
30+
expect(
31+
parseCodePageRegistryOutput(
32+
"ERROR: The system was unable to find the specified registry key or value.",
33+
),
34+
).toBeNull();
35+
});
36+
});
37+
38+
describe("windowsRegistryTool", () => {
39+
it("resolves reg.exe from the Windows install instead of the working directory", () => {
40+
expect(windowsRegistryTool({ SystemRoot: "D:\\Windows" })).toBe(
41+
"D:\\Windows\\System32\\reg.exe",
42+
);
43+
expect(windowsRegistryTool({})).toBe("C:\\Windows\\System32\\reg.exe");
44+
});
45+
});
46+
47+
describe("ansiEncodingForCodePage", () => {
48+
it("maps single and double byte code pages to TextDecoder labels", () => {
49+
expect(ansiEncodingForCodePage(1252)).toBe("windows-1252");
50+
expect(ansiEncodingForCodePage(1251)).toBe("windows-1251");
51+
expect(ansiEncodingForCodePage(1250)).toBe("windows-1250");
52+
expect(ansiEncodingForCodePage(936)).toBe("gbk");
53+
expect(ansiEncodingForCodePage(932)).toBe("shift_jis");
54+
expect(ansiEncodingForCodePage(950)).toBe("big5");
55+
});
56+
57+
it("leaves UTF-8 and unknown code pages undecoded", () => {
58+
expect(ansiEncodingForCodePage(65001)).toBeNull();
59+
expect(ansiEncodingForCodePage(42)).toBeNull();
60+
});
61+
});
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
// @effect-diagnostics nodeBuiltinImport:off -- The ANSI code page is only readable through a host probe.
2+
import * as Context from "effect/Context";
3+
import * as Effect from "effect/Effect";
4+
import * as NodeChildProcess from "node:child_process";
5+
import * as NodePath from "node:path";
6+
7+
import * as HostProcess from "./HostProcess.ts";
8+
9+
// Windows CLIs that Python does not run in UTF-8 mode (az, aws) encode piped output in the
10+
// system ANSI code page, so bytes above 0x7f are not UTF-8.
11+
const ANSI_ENCODINGS_BY_CODE_PAGE: Readonly<Record<number, string>> = {
12+
874: "windows-874",
13+
932: "shift_jis",
14+
936: "gbk",
15+
949: "euc-kr",
16+
950: "big5",
17+
1250: "windows-1250",
18+
1251: "windows-1251",
19+
1252: "windows-1252",
20+
1253: "windows-1253",
21+
1254: "windows-1254",
22+
1255: "windows-1255",
23+
1256: "windows-1256",
24+
1257: "windows-1257",
25+
1258: "windows-1258",
26+
};
27+
28+
/** TextDecoder label for a Windows ANSI code page, or null when its bytes have no text form. */
29+
export const ansiEncodingForCodePage = (codePage: number): string | null =>
30+
ANSI_ENCODINGS_BY_CODE_PAGE[codePage] ?? null;
31+
32+
const CODE_PAGE_REGISTRY_KEY = "HKLM\\SYSTEM\\CurrentControlSet\\Control\\Nls\\CodePage";
33+
34+
export const parseCodePageRegistryOutput = (output: string): number | null => {
35+
const match = /^\s*ACP\s+REG_SZ\s+(\d+)\s*$/mu.exec(output);
36+
return match === null ? null : Number(match[1]);
37+
};
38+
39+
// A bare `reg` is resolved against the current directory before PATH on Windows, so a checkout
40+
// could supply it; ask the Windows install that started this process instead.
41+
export const windowsRegistryTool = (env: NodeJS.ProcessEnv): string =>
42+
NodePath.win32.join(env.SystemRoot ?? "C:\\Windows", "System32", "reg.exe");
43+
44+
const queryCodePage = (regPath: string): Promise<string | null> =>
45+
new Promise((resolve) => {
46+
try {
47+
NodeChildProcess.execFile(
48+
regPath,
49+
["query", CODE_PAGE_REGISTRY_KEY, "/v", "ACP"],
50+
{ encoding: "utf8", timeout: 2_000, windowsHide: true },
51+
(error, stdout) => resolve(error === null ? stdout : null),
52+
);
53+
} catch {
54+
resolve(null);
55+
}
56+
});
57+
58+
const readWindowsAnsiEncoding = Effect.fn("windowsCodePage.readWindowsAnsiEncoding")(function* () {
59+
const environment = yield* HostProcess.Environment;
60+
const output = yield* Effect.tryPromise(() =>
61+
queryCodePage(windowsRegistryTool(environment)),
62+
).pipe(Effect.orElseSucceed(() => null));
63+
const codePage = output === null ? null : parseCodePageRegistryOutput(output);
64+
return codePage === null ? null : ansiEncodingForCodePage(codePage);
65+
});
66+
67+
export const WindowsAnsiEncoding = Context.Reference<Effect.Effect<string | null>>(
68+
"@t3tools/shared/windowsCodePage/AnsiEncoding",
69+
{ defaultValue: readWindowsAnsiEncoding },
70+
);

0 commit comments

Comments
 (0)