Skip to content

Commit 20110b7

Browse files
committed
fix(server): read Windows CLI output in UTF-8, not the ANSI code page
Python CLIs such as az are not in UTF-8 mode, so a piped stdout is encoded in the system ANSI code page: every non-ASCII character in Azure DevOps PR text rendered as U+FFFD. Worse, the ANSI code page cannot carry every character, and az then drops all non-ASCII text of any response that holds one, which loses titles outright. Run the Azure CLI's own interpreter with -X utf8 when it is the MSI layout, so az writes UTF-8 and neither failure happens. -X is an argv flag, so the launcher's -I/-E cannot suppress it the way it does PYTHONUTF8. Installations T3 cannot upgrade keep the documented fallback: decode output that is not valid UTF-8 with the host ANSI code page, opting in per command through ProcessRunInput.windowsAnsiOutput. A truncated buffer has a cut multi-byte tail ignored before the encoding is judged. The code page query runs %SystemRoot%\\System32\\reg.exe rather than a bare reg.
1 parent 0b02b21 commit 20110b7

13 files changed

Lines changed: 497 additions & 9 deletions

File tree

‎apps/server/src/processRunner.test.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import * as Stream from "effect/Stream";
88
import { ChildProcessSpawner } from "effect/process";
99
import * as HostProcess from "@t3tools/shared/HostProcess";
1010
import { SpawnExecutableResolution } from "@t3tools/shared/shell";
11+
import { WindowsAnsiEncoding } from "@t3tools/shared/windowsCodePage";
1112

1213
import * as ProcessRunner from "./processRunner.ts";
1314

@@ -231,6 +232,60 @@ describe("runProcess", () => {
231232
}),
232233
);
233234

235+
it.effect("decodes Windows CLI output written in the ANSI code page", () => {
236+
const spawner = makeSpawner(() =>
237+
Effect.succeed(makeHandle({ stdout: Stream.make(new Uint8Array([0x63, 0x61, 0x66, 0xe9])) })),
238+
);
239+
240+
return runWith(spawner)({
241+
command: "az",
242+
args: ["repos", "pr", "show"],
243+
windowsAnsiOutput: true,
244+
}).pipe(
245+
Effect.provideService(HostProcess.Platform, "win32"),
246+
Effect.provideService(WindowsAnsiEncoding, Effect.succeed("windows-1252")),
247+
Effect.map((result) => {
248+
expect(result.stdout).toBe("caf\u00e9");
249+
expect(result.stdoutInvalidUtf8).toBe(true);
250+
}),
251+
);
252+
});
253+
254+
it.effect("keeps invalid UTF-8 for a command that did not opt in", () => {
255+
const spawner = makeSpawner(() =>
256+
Effect.succeed(makeHandle({ stdout: Stream.make(new Uint8Array([0x63, 0x61, 0x66, 0xe9])) })),
257+
);
258+
259+
return runWith(spawner)({
260+
command: "git",
261+
args: ["diff"],
262+
}).pipe(
263+
Effect.provideService(HostProcess.Platform, "win32"),
264+
Effect.provideService(WindowsAnsiEncoding, Effect.succeed("windows-1252")),
265+
Effect.map((result) => {
266+
expect(result.stdout).toBe("caf\uFFFD");
267+
}),
268+
);
269+
});
270+
271+
it.effect("keeps invalid UTF-8 outside Windows", () => {
272+
const spawner = makeSpawner(() =>
273+
Effect.succeed(makeHandle({ stdout: Stream.make(new Uint8Array([0x63, 0x61, 0x66, 0xe9])) })),
274+
);
275+
276+
return runWith(spawner)({
277+
command: "az",
278+
args: ["repos", "pr", "show"],
279+
windowsAnsiOutput: true,
280+
}).pipe(
281+
Effect.provideService(HostProcess.Platform, "linux"),
282+
Effect.provideService(WindowsAnsiEncoding, Effect.succeed("windows-1252")),
283+
Effect.map((result) => {
284+
expect(result.stdout).toBe("caf\uFFFD");
285+
}),
286+
);
287+
});
288+
234289
it.effect("writes stdin before waiting for exit", () =>
235290
Effect.gen(function* () {
236291
const stdinWritten = yield* Deferred.make<void>();

‎apps/server/src/processRunner.ts‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@ import * as Scope from "effect/Scope";
99
import * as Stream from "effect/Stream";
1010
import * as ChildProcess from "effect/process/ChildProcess";
1111
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
12+
import * as HostProcess from "@t3tools/shared/HostProcess";
1213
import { resolveSpawnCommand } from "@t3tools/shared/shell";
14+
import { WindowsAnsiEncoding } from "@t3tools/shared/windowsCodePage";
1315
import {
1416
collectUint8StreamText,
1517
decodeUtf8,
@@ -29,6 +31,11 @@ export interface ProcessRunInput {
2931
readonly maxOutputBytes?: number | undefined;
3032
readonly outputMode?: "error" | "truncate" | undefined;
3133
readonly truncatedMarker?: string | undefined;
34+
/**
35+
* Windows CLIs that are not in UTF-8 mode (`az`) write piped output in the ANSI code page.
36+
* Opt in per command so a UTF-8 tool with a stray legacy byte keeps its text.
37+
*/
38+
readonly windowsAnsiOutput?: boolean | undefined;
3239
/** Timed-out results discard partial stdout/stderr. */
3340
readonly timeoutBehavior?: "error" | "timedOutResult" | undefined;
3441
}
@@ -159,6 +166,7 @@ const collectText = Effect.fnUntraced(function* (input: {
159166
readonly maxOutputBytes: number;
160167
readonly outputMode: "error" | "truncate";
161168
readonly truncatedMarker: string;
169+
readonly legacyEncoding: string | null;
162170
}) {
163171
const stream = input.stream.pipe(
164172
Stream.mapError(
@@ -179,6 +187,7 @@ const collectText = Effect.fnUntraced(function* (input: {
179187
stream,
180188
maxBytes: input.maxOutputBytes,
181189
truncatedMarker: input.truncatedMarker,
190+
legacyEncoding: input.legacyEncoding,
182191
});
183192
}
184193

@@ -217,7 +226,9 @@ const collectText = Effect.fnUntraced(function* (input: {
217226
},
218227
),
219228
Effect.map((state): CollectedUint8StreamText => ({
220-
...decodeUtf8(Buffer.concat(state.chunks, state.bytes)),
229+
...decodeUtf8(Buffer.concat(state.chunks, state.bytes), {
230+
legacyEncoding: input.legacyEncoding,
231+
}),
221232
bytes: state.bytes,
222233
truncated: false,
223234
})),
@@ -267,12 +278,14 @@ const commandName = (command: string) => command.replace(/^.*[\\/]/, "");
267278

268279
const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
269280
spawner: ChildProcessSpawner.ChildProcessSpawner["Service"],
281+
legacyOutputEncoding: Effect.Effect<string | null>,
270282
input: ProcessRunInput,
271283
): Effect.fn.Return<ProcessRunOutput, ProcessRunError, Scope.Scope> {
272284
yield* Effect.annotateCurrentSpan("process.command", commandName(input.command));
273285
const maxOutputBytes = input.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
274286
const outputMode = input.outputMode ?? "error";
275287
const truncatedMarker = input.truncatedMarker ?? "";
288+
const legacyEncoding = input.windowsAnsiOutput === true ? yield* legacyOutputEncoding : null;
276289
const extendEnv = input.env !== undefined;
277290
const spawnCommand = yield* resolveSpawnCommand(
278291
input.command,
@@ -345,6 +358,7 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
345358
maxOutputBytes,
346359
outputMode,
347360
truncatedMarker,
361+
legacyEncoding,
348362
}),
349363
collectText({
350364
command: input.command,
@@ -356,6 +370,7 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
356370
maxOutputBytes,
357371
outputMode,
358372
truncatedMarker,
373+
legacyEncoding,
359374
}),
360375
writeStdin,
361376
],
@@ -391,9 +406,14 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
391406
/** @public */
392407
export const make = Effect.fn("ProcessRunner.make")(function* () {
393408
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
409+
const platform = yield* HostProcess.Platform;
410+
// Python CLIs that are not in UTF-8 mode (`az`) encode piped output in the ANSI code page.
411+
const legacyEncoding: Effect.Effect<string | null> =
412+
platform === "win32" ? yield* WindowsAnsiEncoding : Effect.succeed(null);
413+
const legacyOutputEncoding = yield* Effect.cached(legacyEncoding);
394414

395415
const run: ProcessRunner["Service"]["run"] = (input) =>
396-
finalizeRunProcess(runProcessCore(spawner, input), input);
416+
finalizeRunProcess(runProcessCore(spawner, legacyOutputEncoding, input), input);
397417

398418
return ProcessRunner.of({
399419
run,

‎apps/server/src/vcs/VcsProcess.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ export interface VcsProcessInput {
3939
readonly maxOutputBytes?: number;
4040
readonly outputMode?: ProcessRunner.ProcessRunInput["outputMode"];
4141
readonly appendTruncationMarker?: boolean;
42+
readonly windowsAnsiOutput?: boolean | undefined;
4243
readonly classifyFailure?: SourceControlProcessInput["classifyFailure"];
4344
readonly notFoundDetail?: string;
4445
readonly concurrency?: SourceControlProcessInput["concurrency"];
@@ -215,6 +216,7 @@ export const make = Effect.gen(function* () {
215216
maxOutputBytes: input.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES,
216217
outputMode: input.outputMode ?? "truncate",
217218
truncatedMarker: input.appendTruncationMarker ? OUTPUT_TRUNCATED_MARKER : "",
219+
windowsAnsiOutput: input.windowsAnsiOutput,
218220
timeoutBehavior: "error",
219221
})
220222
.pipe(

‎packages/provider-core/src/server/collectStreamText.test.ts‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,4 +39,93 @@ describe("collectUint8StreamText", () => {
3939
assert.strictEqual(literal.text, "before\uFFFDafter");
4040
}),
4141
);
42+
43+
it.effect("decodes ANSI-encoded output with the given legacy encoding", () =>
44+
Effect.gen(function* () {
45+
const result = yield* collectUint8StreamText({
46+
stream: Stream.make(new Uint8Array([0x63, 0x61, 0x66, 0xe9])),
47+
legacyEncoding: "windows-1252",
48+
});
49+
50+
assert.strictEqual(result.text, "caf\u00e9");
51+
assert.strictEqual(result.invalidUtf8, true);
52+
}),
53+
);
54+
55+
it.effect("keeps a truncated multi-byte character a replacement character", () =>
56+
Effect.gen(function* () {
57+
const result = yield* collectUint8StreamText({
58+
stream: Stream.make(encoder.encode("caf\u00e9"), encoder.encode(" next")),
59+
maxBytes: 4,
60+
truncatedMarker: "[truncated]",
61+
legacyEncoding: "windows-1252",
62+
});
63+
64+
assert.strictEqual(result.text, "caf\uFFFD[truncated]");
65+
assert.strictEqual(result.invalidUtf8, true);
66+
}),
67+
);
68+
69+
it.effect("decodes truncated ANSI output, not just its cut tail", () =>
70+
Effect.gen(function* () {
71+
// "Café título" in windows-1252, cut before its last letter.
72+
const ansi = new Uint8Array([
73+
0x43, 0x61, 0x66, 0xe9, 0x20, 0x74, 0xed, 0x74, 0x75, 0x6c, 0x6f,
74+
]);
75+
const result = yield* collectUint8StreamText({
76+
stream: Stream.make(ansi, encoder.encode(" rest")),
77+
maxBytes: ansi.byteLength,
78+
truncatedMarker: "[truncated]",
79+
legacyEncoding: "windows-1252",
80+
});
81+
82+
assert.strictEqual(result.text, "Caf\u00e9 t\u00edtulo[truncated]");
83+
assert.strictEqual(result.truncated, true);
84+
assert.strictEqual(result.invalidUtf8, true);
85+
}),
86+
);
87+
88+
it.effect("keeps ANSI text when a legacy byte sits inside a truncated buffer", () =>
89+
Effect.gen(function* () {
90+
// "café!" in windows-1252, cut by our own byte limit.
91+
const ansi = new Uint8Array([0x63, 0x61, 0x66, 0xe9, 0x21]);
92+
const result = yield* collectUint8StreamText({
93+
stream: Stream.make(ansi, encoder.encode(" more")),
94+
maxBytes: ansi.byteLength,
95+
legacyEncoding: "windows-1252",
96+
});
97+
98+
assert.strictEqual(result.text, "caf\u00e9!");
99+
assert.strictEqual(result.truncated, true);
100+
}),
101+
);
102+
103+
it.effect("keeps ANSI bytes that cannot start the UTF-8 sequence they look like", () =>
104+
Effect.gen(function* () {
105+
// "à€" in windows-1252: e0 is a three-byte lead, but e0 80 is not a valid prefix of one.
106+
const ansi = new Uint8Array([0xe0, 0x80]);
107+
const result = yield* collectUint8StreamText({
108+
stream: Stream.make(ansi, encoder.encode(" more")),
109+
maxBytes: ansi.byteLength,
110+
legacyEncoding: "windows-1252",
111+
});
112+
113+
assert.strictEqual(result.text, "\u00e0\u20ac");
114+
assert.strictEqual(result.truncated, true);
115+
}),
116+
);
117+
118+
it.effect("still treats a cut four-byte character as incomplete", () =>
119+
Effect.gen(function* () {
120+
const cut = new Uint8Array([0x68, 0x69, 0xf0, 0x9f, 0x92]);
121+
const result = yield* collectUint8StreamText({
122+
stream: Stream.make(cut, encoder.encode(" more")),
123+
maxBytes: cut.byteLength,
124+
truncatedMarker: "[truncated]",
125+
legacyEncoding: "windows-1252",
126+
});
127+
128+
assert.strictEqual(result.text, "hi\uFFFD[truncated]");
129+
}),
130+
);
42131
});

‎packages/provider-core/src/server/collectStreamText.ts‎

Lines changed: 44 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,47 @@ export interface CollectedUint8StreamText {
99
readonly invalidUtf8: boolean;
1010
}
1111

12+
export interface DecodeStreamTextOptions {
13+
/** Decodes output that is not UTF-8, such as a Windows CLI writing its ANSI code page. */
14+
readonly legacyEncoding?: string | null | undefined;
15+
/** Our own byte limit can cut a multi-byte character, so that tail is no evidence of another encoding. */
16+
readonly truncated?: boolean | undefined;
17+
}
18+
19+
const decodeWithEncoding = (bytes: Uint8Array, encoding: string): string => {
20+
try {
21+
return new TextDecoder(encoding).decode(bytes);
22+
} catch {
23+
return Buffer.from(bytes).toString("utf8");
24+
}
25+
};
26+
27+
/** Stream mode holds a multi-byte character our byte limit cut, instead of rejecting the buffer. */
28+
const isUtf8Prefix = (bytes: Uint8Array): boolean => {
29+
try {
30+
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes, { stream: true });
31+
return true;
32+
} catch {
33+
return false;
34+
}
35+
};
36+
1237
export const decodeUtf8 = (
1338
bytes: Uint8Array,
14-
): Pick<CollectedUint8StreamText, "text" | "invalidUtf8"> => ({
15-
text: Buffer.from(bytes).toString("utf8"),
16-
invalidUtf8: !NodeBuffer.isUtf8(bytes),
17-
});
39+
options: DecodeStreamTextOptions = {},
40+
): Pick<CollectedUint8StreamText, "text" | "invalidUtf8"> => {
41+
const invalidUtf8 = !NodeBuffer.isUtf8(bytes);
42+
const legacyEncoding = options.legacyEncoding ?? null;
43+
const decodable = options.truncated === true ? isUtf8Prefix(bytes) : !invalidUtf8;
44+
45+
return {
46+
text:
47+
legacyEncoding !== null && !decodable
48+
? decodeWithEncoding(bytes, legacyEncoding)
49+
: Buffer.from(bytes).toString("utf8"),
50+
invalidUtf8,
51+
};
52+
};
1853

1954
interface CollectState {
2055
chunks: Uint8Array[];
@@ -26,6 +61,7 @@ export const collectUint8StreamText = <E>(input: {
2661
readonly stream: Stream.Stream<Uint8Array, E>;
2762
readonly maxBytes?: number | undefined;
2863
readonly truncatedMarker?: string | null | undefined;
64+
readonly legacyEncoding?: string | null | undefined;
2965
}): Effect.Effect<CollectedUint8StreamText, E> => {
3066
const maxBytes = input.maxBytes ?? Number.POSITIVE_INFINITY;
3167
const truncatedMarker = input.truncatedMarker ?? "";
@@ -65,7 +101,10 @@ export const collectUint8StreamText = <E>(input: {
65101
},
66102
),
67103
Effect.map((state): CollectedUint8StreamText => {
68-
const decoded = decodeUtf8(Buffer.concat(state.chunks, state.bytes));
104+
const decoded = decodeUtf8(Buffer.concat(state.chunks, state.bytes), {
105+
legacyEncoding: input.legacyEncoding,
106+
truncated: state.truncated,
107+
});
69108
return {
70109
text:
71110
state.truncated && truncatedMarker.length > 0

‎packages/shared/package.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -502,6 +502,10 @@
502502
"./cloneUrl": {
503503
"types": "./src/cloneUrl.ts",
504504
"import": "./src/cloneUrl.ts"
505+
},
506+
"./windowsCodePage": {
507+
"types": "./src/windowsCodePage.ts",
508+
"import": "./src/windowsCodePage.ts"
505509
}
506510
},
507511
"scripts": {

0 commit comments

Comments
 (0)