Skip to content

Commit f9ecee4

Browse files
committed
fix(server): decode Windows CLI output written in the ANSI code page
Python CLIs such as az are not in UTF-8 mode, so their piped stdout is encoded in the system ANSI code page. Every non-ASCII character in Azure DevOps PR titles, descriptions and comments rendered as U+FFFD because process output was decoded as UTF-8 only. Decode a buffer that is not valid UTF-8 with the ANSI code page the host reports, for the commands that opt in: ProcessRunInput.windowsAnsiOutput, set by the Azure DevOps CLI. A UTF-8 tool with a stray legacy byte, such as a git diff over a latin-1 file, keeps its text and its replacement characters. A truncated buffer has its cut multi-byte tail ignored before the encoding is judged, so long ANSI output still decodes. The code page query runs %SystemRoot%\System32\reg.exe rather than a bare reg, so the current directory cannot supply the executable.
1 parent 0b02b21 commit f9ecee4

11 files changed

Lines changed: 387 additions & 7 deletions

File tree

‎apps/server/src/processRunner.test.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import * as Stream from "effect/Stream";
88
import { ChildProcessSpawner } from "effect/process";
99
import * as HostProcess from "@t3tools/shared/HostProcess";
1010
import { SpawnExecutableResolution } from "@t3tools/shared/shell";
11+
import { WindowsAnsiEncoding } from "@t3tools/shared/windowsCodePage";
1112

1213
import * as ProcessRunner from "./processRunner.ts";
1314

@@ -231,6 +232,60 @@ describe("runProcess", () => {
231232
}),
232233
);
233234

235+
it.effect("decodes Windows CLI output written in the ANSI code page", () => {
236+
const spawner = makeSpawner(() =>
237+
Effect.succeed(makeHandle({ stdout: Stream.make(new Uint8Array([0x63, 0x61, 0x66, 0xe9])) })),
238+
);
239+
240+
return runWith(spawner)({
241+
command: "az",
242+
args: ["repos", "pr", "show"],
243+
windowsAnsiOutput: true,
244+
}).pipe(
245+
Effect.provideService(HostProcess.Platform, "win32"),
246+
Effect.provideService(WindowsAnsiEncoding, Effect.succeed("windows-1252")),
247+
Effect.map((result) => {
248+
expect(result.stdout).toBe("caf\u00e9");
249+
expect(result.stdoutInvalidUtf8).toBe(true);
250+
}),
251+
);
252+
});
253+
254+
it.effect("keeps invalid UTF-8 for a command that did not opt in", () => {
255+
const spawner = makeSpawner(() =>
256+
Effect.succeed(makeHandle({ stdout: Stream.make(new Uint8Array([0x63, 0x61, 0x66, 0xe9])) })),
257+
);
258+
259+
return runWith(spawner)({
260+
command: "git",
261+
args: ["diff"],
262+
}).pipe(
263+
Effect.provideService(HostProcess.Platform, "win32"),
264+
Effect.provideService(WindowsAnsiEncoding, Effect.succeed("windows-1252")),
265+
Effect.map((result) => {
266+
expect(result.stdout).toBe("caf\uFFFD");
267+
}),
268+
);
269+
});
270+
271+
it.effect("keeps invalid UTF-8 outside Windows", () => {
272+
const spawner = makeSpawner(() =>
273+
Effect.succeed(makeHandle({ stdout: Stream.make(new Uint8Array([0x63, 0x61, 0x66, 0xe9])) })),
274+
);
275+
276+
return runWith(spawner)({
277+
command: "az",
278+
args: ["repos", "pr", "show"],
279+
windowsAnsiOutput: true,
280+
}).pipe(
281+
Effect.provideService(HostProcess.Platform, "linux"),
282+
Effect.provideService(WindowsAnsiEncoding, Effect.succeed("windows-1252")),
283+
Effect.map((result) => {
284+
expect(result.stdout).toBe("caf\uFFFD");
285+
}),
286+
);
287+
});
288+
234289
it.effect("writes stdin before waiting for exit", () =>
235290
Effect.gen(function* () {
236291
const stdinWritten = yield* Deferred.make<void>();

‎apps/server/src/processRunner.ts‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@ import * as Scope from "effect/Scope";
99
import * as Stream from "effect/Stream";
1010
import * as ChildProcess from "effect/process/ChildProcess";
1111
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
12+
import * as HostProcess from "@t3tools/shared/HostProcess";
1213
import { resolveSpawnCommand } from "@t3tools/shared/shell";
14+
import { WindowsAnsiEncoding } from "@t3tools/shared/windowsCodePage";
1315
import {
1416
collectUint8StreamText,
1517
decodeUtf8,
@@ -29,6 +31,11 @@ export interface ProcessRunInput {
2931
readonly maxOutputBytes?: number | undefined;
3032
readonly outputMode?: "error" | "truncate" | undefined;
3133
readonly truncatedMarker?: string | undefined;
34+
/**
35+
* Windows CLIs that are not in UTF-8 mode (`az`) write piped output in the ANSI code page.
36+
* Opt in per command so a UTF-8 tool with a stray legacy byte keeps its text.
37+
*/
38+
readonly windowsAnsiOutput?: boolean | undefined;
3239
/** Timed-out results discard partial stdout/stderr. */
3340
readonly timeoutBehavior?: "error" | "timedOutResult" | undefined;
3441
}
@@ -159,6 +166,7 @@ const collectText = Effect.fnUntraced(function* (input: {
159166
readonly maxOutputBytes: number;
160167
readonly outputMode: "error" | "truncate";
161168
readonly truncatedMarker: string;
169+
readonly legacyEncoding: string | null;
162170
}) {
163171
const stream = input.stream.pipe(
164172
Stream.mapError(
@@ -179,6 +187,7 @@ const collectText = Effect.fnUntraced(function* (input: {
179187
stream,
180188
maxBytes: input.maxOutputBytes,
181189
truncatedMarker: input.truncatedMarker,
190+
legacyEncoding: input.legacyEncoding,
182191
});
183192
}
184193

@@ -217,7 +226,9 @@ const collectText = Effect.fnUntraced(function* (input: {
217226
},
218227
),
219228
Effect.map((state): CollectedUint8StreamText => ({
220-
...decodeUtf8(Buffer.concat(state.chunks, state.bytes)),
229+
...decodeUtf8(Buffer.concat(state.chunks, state.bytes), {
230+
legacyEncoding: input.legacyEncoding,
231+
}),
221232
bytes: state.bytes,
222233
truncated: false,
223234
})),
@@ -267,12 +278,15 @@ const commandName = (command: string) => command.replace(/^.*[\\/]/, "");
267278

268279
const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
269280
spawner: ChildProcessSpawner.ChildProcessSpawner["Service"],
281+
legacyOutputEncoding: Effect.Effect<string | null>,
270282
input: ProcessRunInput,
271283
): Effect.fn.Return<ProcessRunOutput, ProcessRunError, Scope.Scope> {
272284
yield* Effect.annotateCurrentSpan("process.command", commandName(input.command));
273285
const maxOutputBytes = input.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
274286
const outputMode = input.outputMode ?? "error";
275287
const truncatedMarker = input.truncatedMarker ?? "";
288+
// The probe is cached, so the first `az` run on Windows pays its one query.
289+
const legacyEncoding = input.windowsAnsiOutput === true ? yield* legacyOutputEncoding : null;
276290
const extendEnv = input.env !== undefined;
277291
const spawnCommand = yield* resolveSpawnCommand(
278292
input.command,
@@ -345,6 +359,7 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
345359
maxOutputBytes,
346360
outputMode,
347361
truncatedMarker,
362+
legacyEncoding,
348363
}),
349364
collectText({
350365
command: input.command,
@@ -356,6 +371,7 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
356371
maxOutputBytes,
357372
outputMode,
358373
truncatedMarker,
374+
legacyEncoding,
359375
}),
360376
writeStdin,
361377
],
@@ -391,9 +407,14 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
391407
/** @public */
392408
export const make = Effect.fn("ProcessRunner.make")(function* () {
393409
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
410+
const platform = yield* HostProcess.Platform;
411+
// Python CLIs that are not in UTF-8 mode (`az`) encode piped output in the ANSI code page.
412+
const legacyEncoding: Effect.Effect<string | null> =
413+
platform === "win32" ? yield* WindowsAnsiEncoding : Effect.succeed(null);
414+
const legacyOutputEncoding = yield* Effect.cached(legacyEncoding);
394415

395416
const run: ProcessRunner["Service"]["run"] = (input) =>
396-
finalizeRunProcess(runProcessCore(spawner, input), input);
417+
finalizeRunProcess(runProcessCore(spawner, legacyOutputEncoding, input), input);
397418

398419
return ProcessRunner.of({
399420
run,

‎apps/server/src/vcs/VcsProcess.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ export interface VcsProcessInput {
3939
readonly maxOutputBytes?: number;
4040
readonly outputMode?: ProcessRunner.ProcessRunInput["outputMode"];
4141
readonly appendTruncationMarker?: boolean;
42+
readonly windowsAnsiOutput?: ProcessRunner.ProcessRunInput["windowsAnsiOutput"];
4243
readonly classifyFailure?: SourceControlProcessInput["classifyFailure"];
4344
readonly notFoundDetail?: string;
4445
readonly concurrency?: SourceControlProcessInput["concurrency"];
@@ -215,6 +216,9 @@ export const make = Effect.gen(function* () {
215216
maxOutputBytes: input.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES,
216217
outputMode: input.outputMode ?? "truncate",
217218
truncatedMarker: input.appendTruncationMarker ? OUTPUT_TRUNCATED_MARKER : "",
219+
...(input.windowsAnsiOutput === undefined
220+
? {}
221+
: { windowsAnsiOutput: input.windowsAnsiOutput }),
218222
timeoutBehavior: "error",
219223
})
220224
.pipe(

‎packages/provider-core/src/server/collectStreamText.test.ts‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,4 +39,93 @@ describe("collectUint8StreamText", () => {
3939
assert.strictEqual(literal.text, "before\uFFFDafter");
4040
}),
4141
);
42+
43+
it.effect("decodes ANSI-encoded output with the given legacy encoding", () =>
44+
Effect.gen(function* () {
45+
const result = yield* collectUint8StreamText({
46+
stream: Stream.make(new Uint8Array([0x63, 0x61, 0x66, 0xe9])),
47+
legacyEncoding: "windows-1252",
48+
});
49+
50+
assert.strictEqual(result.text, "caf\u00e9");
51+
assert.strictEqual(result.invalidUtf8, true);
52+
}),
53+
);
54+
55+
it.effect("keeps a truncated multi-byte character a replacement character", () =>
56+
Effect.gen(function* () {
57+
const result = yield* collectUint8StreamText({
58+
stream: Stream.make(encoder.encode("caf\u00e9"), encoder.encode(" next")),
59+
maxBytes: 4,
60+
truncatedMarker: "[truncated]",
61+
legacyEncoding: "windows-1252",
62+
});
63+
64+
assert.strictEqual(result.text, "caf\uFFFD[truncated]");
65+
assert.strictEqual(result.invalidUtf8, true);
66+
}),
67+
);
68+
69+
it.effect("decodes truncated ANSI output, not just its cut tail", () =>
70+
Effect.gen(function* () {
71+
// "Café título" in windows-1252, cut before its last letter.
72+
const ansi = new Uint8Array([
73+
0x43, 0x61, 0x66, 0xe9, 0x20, 0x74, 0xed, 0x74, 0x75, 0x6c, 0x6f,
74+
]);
75+
const result = yield* collectUint8StreamText({
76+
stream: Stream.make(ansi, encoder.encode(" rest")),
77+
maxBytes: ansi.byteLength,
78+
truncatedMarker: "[truncated]",
79+
legacyEncoding: "windows-1252",
80+
});
81+
82+
assert.strictEqual(result.text, "Caf\u00e9 t\u00edtulo[truncated]");
83+
assert.strictEqual(result.truncated, true);
84+
assert.strictEqual(result.invalidUtf8, true);
85+
}),
86+
);
87+
88+
it.effect("keeps truncated ANSI text whose tail also passes as a cut UTF-8 sequence", () =>
89+
Effect.gen(function* () {
90+
// "café!" in windows-1252: dropping e9 21 would leave valid UTF-8, but nothing here is a cut character.
91+
const ansi = new Uint8Array([0x63, 0x61, 0x66, 0xe9, 0x21]);
92+
const result = yield* collectUint8StreamText({
93+
stream: Stream.make(ansi, encoder.encode(" more")),
94+
maxBytes: ansi.byteLength,
95+
legacyEncoding: "windows-1252",
96+
});
97+
98+
assert.strictEqual(result.text, "caf\u00e9!");
99+
assert.strictEqual(result.truncated, true);
100+
}),
101+
);
102+
103+
it.effect("keeps ANSI bytes that cannot start the UTF-8 sequence they look like", () =>
104+
Effect.gen(function* () {
105+
// "à€" in windows-1252: e0 is a three-byte lead, but e0 80 is not a valid prefix of one.
106+
const ansi = new Uint8Array([0xe0, 0x80]);
107+
const result = yield* collectUint8StreamText({
108+
stream: Stream.make(ansi, encoder.encode(" more")),
109+
maxBytes: ansi.byteLength,
110+
legacyEncoding: "windows-1252",
111+
});
112+
113+
assert.strictEqual(result.text, "\u00e0\u20ac");
114+
assert.strictEqual(result.truncated, true);
115+
}),
116+
);
117+
118+
it.effect("still treats a cut four-byte character as incomplete", () =>
119+
Effect.gen(function* () {
120+
const cut = new Uint8Array([0x68, 0x69, 0xf0, 0x9f, 0x92]);
121+
const result = yield* collectUint8StreamText({
122+
stream: Stream.make(cut, encoder.encode(" more")),
123+
maxBytes: cut.byteLength,
124+
truncatedMarker: "[truncated]",
125+
legacyEncoding: "windows-1252",
126+
});
127+
128+
assert.strictEqual(result.text, "hi\uFFFD[truncated]");
129+
}),
130+
);
42131
});

‎packages/provider-core/src/server/collectStreamText.ts‎

Lines changed: 76 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,79 @@ export interface CollectedUint8StreamText {
99
readonly invalidUtf8: boolean;
1010
}
1111

12+
export interface DecodeStreamTextOptions {
13+
/** Decodes output that is not UTF-8, such as a Windows CLI writing its ANSI code page. */
14+
readonly legacyEncoding?: string | null | undefined;
15+
/** Our own byte limit can cut a multi-byte character, so that tail is no evidence of another encoding. */
16+
readonly truncated?: boolean | undefined;
17+
}
18+
19+
const decodeWithEncoding = (bytes: Uint8Array, encoding: string): string => {
20+
try {
21+
return new TextDecoder(encoding).decode(bytes);
22+
} catch {
23+
return Buffer.from(bytes).toString("utf8");
24+
}
25+
};
26+
27+
interface Utf8Sequence {
28+
readonly length: number;
29+
/** Range the second byte may take, which strict UTF-8 narrows below the continuation range. */
30+
readonly secondMin: number;
31+
readonly secondMax: number;
32+
}
33+
34+
const utf8SequenceForLead = (lead: number): Utf8Sequence | null => {
35+
if (lead >= 0xc2 && lead <= 0xdf) return { length: 2, secondMin: 0x80, secondMax: 0xbf };
36+
if (lead === 0xe0) return { length: 3, secondMin: 0xa0, secondMax: 0xbf };
37+
if (lead >= 0xe1 && lead <= 0xec) return { length: 3, secondMin: 0x80, secondMax: 0xbf };
38+
if (lead === 0xed) return { length: 3, secondMin: 0x80, secondMax: 0x9f };
39+
if (lead >= 0xee && lead <= 0xef) return { length: 3, secondMin: 0x80, secondMax: 0xbf };
40+
if (lead === 0xf0) return { length: 4, secondMin: 0x90, secondMax: 0xbf };
41+
if (lead >= 0xf1 && lead <= 0xf3) return { length: 4, secondMin: 0x80, secondMax: 0xbf };
42+
if (lead === 0xf4) return { length: 4, secondMin: 0x80, secondMax: 0x8f };
43+
return null;
44+
};
45+
46+
/**
47+
* Bytes of a trailing UTF-8 sequence our byte limit cut short, or 0 when the tail is complete, has no
48+
* sequence, or holds bytes that could not start one.
49+
*/
50+
const incompleteUtf8TailLength = (bytes: Uint8Array): number => {
51+
let start = bytes.length;
52+
while (start > 0 && bytes.length - start < 3 && (bytes[start - 1]! & 0xc0) === 0x80) start -= 1;
53+
if (start === 0) return 0;
54+
const sequence = utf8SequenceForLead(bytes[start - 1]!);
55+
if (sequence === null) return 0;
56+
const present = bytes.length - start + 1;
57+
if (present >= sequence.length) return 0;
58+
const second = bytes[start];
59+
if (second !== undefined && (second < sequence.secondMin || second > sequence.secondMax))
60+
return 0;
61+
return present;
62+
};
63+
64+
const withoutIncompleteTail = (bytes: Uint8Array): Uint8Array => {
65+
const cut = incompleteUtf8TailLength(bytes);
66+
return cut === 0 ? bytes : bytes.subarray(0, bytes.length - cut);
67+
};
68+
1269
export const decodeUtf8 = (
1370
bytes: Uint8Array,
14-
): Pick<CollectedUint8StreamText, "text" | "invalidUtf8"> => ({
15-
text: Buffer.from(bytes).toString("utf8"),
16-
invalidUtf8: !NodeBuffer.isUtf8(bytes),
17-
});
71+
options: DecodeStreamTextOptions = {},
72+
): Pick<CollectedUint8StreamText, "text" | "invalidUtf8"> => {
73+
const invalidUtf8 = !NodeBuffer.isUtf8(bytes);
74+
const legacyEncoding = options.legacyEncoding ?? null;
75+
const complete = invalidUtf8 && options.truncated === true ? withoutIncompleteTail(bytes) : bytes;
76+
77+
return {
78+
text:
79+
legacyEncoding !== null && !NodeBuffer.isUtf8(complete)
80+
? decodeWithEncoding(bytes, legacyEncoding)
81+
: Buffer.from(bytes).toString("utf8"),
82+
invalidUtf8,
83+
};
84+
};
1885

1986
interface CollectState {
2087
chunks: Uint8Array[];
@@ -26,6 +93,7 @@ export const collectUint8StreamText = <E>(input: {
2693
readonly stream: Stream.Stream<Uint8Array, E>;
2794
readonly maxBytes?: number | undefined;
2895
readonly truncatedMarker?: string | null | undefined;
96+
readonly legacyEncoding?: string | null | undefined;
2997
}): Effect.Effect<CollectedUint8StreamText, E> => {
3098
const maxBytes = input.maxBytes ?? Number.POSITIVE_INFINITY;
3199
const truncatedMarker = input.truncatedMarker ?? "";
@@ -65,7 +133,10 @@ export const collectUint8StreamText = <E>(input: {
65133
},
66134
),
67135
Effect.map((state): CollectedUint8StreamText => {
68-
const decoded = decodeUtf8(Buffer.concat(state.chunks, state.bytes));
136+
const decoded = decodeUtf8(Buffer.concat(state.chunks, state.bytes), {
137+
legacyEncoding: input.legacyEncoding,
138+
truncated: state.truncated,
139+
});
69140
return {
70141
text:
71142
state.truncated && truncatedMarker.length > 0

‎packages/shared/package.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -502,6 +502,10 @@
502502
"./cloneUrl": {
503503
"types": "./src/cloneUrl.ts",
504504
"import": "./src/cloneUrl.ts"
505+
},
506+
"./windowsCodePage": {
507+
"types": "./src/windowsCodePage.ts",
508+
"import": "./src/windowsCodePage.ts"
505509
}
506510
},
507511
"scripts": {

0 commit comments

Comments
 (0)