Skip to content

Commit 4e11da2

Browse files
mushuiyu886steipete
mushuiyu886
and
steipete
authored
fix(process): UTF-8 command output corrupts at Windows byte limits (#105274)
* fix(process): preserve byte-capped UTF-8 output on Windows * fix(process): skip codepage probe for UTF-8 output Co-authored-by: 杨浩宇0668001029 <[email protected]> --------- Co-authored-by: Peter Steinberger <[email protected]>
1 parent 7a551bf commit 4e11da2

7 files changed

Lines changed: 106 additions & 16 deletions

File tree

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1998,7 +1998,7 @@
19981998
"test:unit:fast:audit": "node scripts/test-unit-fast-audit.mjs",
19991999
"test:voicecall:closedloop": "node scripts/test-voicecall-closedloop.mjs",
20002000
"test:watch": "node scripts/test-projects.mjs --watch",
2001-
"test:windows:ci": "node scripts/test-projects.mjs src/shared/runtime-import.test.ts src/config/sessions/store.session-lifecycle-mutation.test.ts src/agents/sessions/windows-git-bash-path.test.ts src/process/exec.windows.test.ts src/process/windows-command.test.ts src/infra/windows-install-roots.test.ts src/node-host/invoke-system-run-allowlist.test.ts src/daemon/schtasks.startup-fallback.test.ts extensions/lobster/src/lobster-runner.test.ts test/scripts/format-generated-module.test.ts test/scripts/npm-runner.test.ts test/scripts/openclaw-cross-os-installer.windows.test.ts test/scripts/openclaw-cross-os-release-workflow.test.ts test/scripts/pnpm-runner.test.ts test/scripts/run-with-env.test.ts test/scripts/ts-topology.test.ts test/scripts/ui.test.ts test/scripts/vitest-process-group.test.ts",
2001+
"test:windows:ci": "node scripts/test-projects.mjs src/shared/runtime-import.test.ts src/config/sessions/store.session-lifecycle-mutation.test.ts src/agents/sessions/windows-git-bash-path.test.ts src/process/exec.windows.test.ts src/process/exec.windows.integration.test.ts src/process/windows-command.test.ts src/infra/windows-install-roots.test.ts src/node-host/invoke-system-run-allowlist.test.ts src/daemon/schtasks.startup-fallback.test.ts extensions/lobster/src/lobster-runner.test.ts test/scripts/format-generated-module.test.ts test/scripts/npm-runner.test.ts test/scripts/openclaw-cross-os-installer.windows.test.ts test/scripts/openclaw-cross-os-release-workflow.test.ts test/scripts/pnpm-runner.test.ts test/scripts/run-with-env.test.ts test/scripts/ts-topology.test.ts test/scripts/ui.test.ts test/scripts/vitest-process-group.test.ts",
20022002
"tool-display:check": "node --import tsx scripts/tool-display.ts --check",
20032003
"tool-display:write": "node --import tsx scripts/tool-display.ts --write",
20042004
"ts-topology": "node --import tsx scripts/ts-topology.ts",

‎src/gateway/node-pairing-ssh-verify.runtime.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
// SSH probe execution for SSH-verified node pairing.
22
// Kept as a narrow runtime boundary so gateway tests can mock the probe
33
// without touching the eligibility/verification policy.
4-
import { runCommandWithTimeout } from "../process/exec.js";
4+
import { runUtf8CommandWithTimeout } from "../process/exec.js";
55

66
export type NodeIdentityProbeParams = {
77
user: string;
@@ -70,7 +70,7 @@ export async function runNodeIdentityProbe(
7070
try {
7171
// PATH-resolved `ssh` keeps Windows OpenSSH working; the gateway process
7272
// environment is operator-owned, so PATH lookup is not an injection risk.
73-
const result = await runCommandWithTimeout(["ssh", ...args], {
73+
const result = await runUtf8CommandWithTimeout(["ssh", ...args], {
7474
maxOutputBytes: MAX_PROBE_OUTPUT_BYTES,
7575
outputCapture: "head",
7676
timeoutMs: Math.max(250, params.timeoutMs),

‎src/process/exec-output.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,8 +113,9 @@ function trimTruncatedUtf8Boundary(
113113
buffer: Buffer,
114114
mode: CommandOutputCaptureMode,
115115
truncatedBytes: number,
116+
forceUtf8: boolean,
116117
): Buffer {
117-
if (truncatedBytes === 0 || buffer.length === 0 || process.platform === "win32") {
118+
if (truncatedBytes === 0 || buffer.length === 0 || (process.platform === "win32" && !forceUtf8)) {
118119
return buffer;
119120
}
120121
if (mode === "tail") {
@@ -140,9 +141,10 @@ function trimTruncatedUtf8Boundary(
140141
export function finalizeCapturedOutput(
141142
capture: CapturedOutputBuffers,
142143
mode: CommandOutputCaptureMode,
144+
forceUtf8 = false,
143145
): Buffer {
144146
const buffered = Buffer.concat(capture.chunks, capture.bytes);
145-
const trimmed = trimTruncatedUtf8Boundary(buffered, mode, capture.truncatedBytes);
147+
const trimmed = trimTruncatedUtf8Boundary(buffered, mode, capture.truncatedBytes, forceUtf8);
146148
capture.truncatedBytes += buffered.byteLength - trimmed.byteLength;
147149
return trimmed;
148150
}

‎src/process/exec-runner.ts‎

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,22 @@ export type CommandOptions = {
6868
export async function runCommandWithTimeout(
6969
argv: string[],
7070
optionsOrTimeout: number | CommandOptions,
71+
): Promise<SpawnResult> {
72+
return await runCommandWithOutputEncoding(argv, optionsOrTimeout, false);
73+
}
74+
75+
/** Run a command whose stdout and stderr are defined to be UTF-8 on every platform. */
76+
export async function runUtf8CommandWithTimeout(
77+
argv: string[],
78+
optionsOrTimeout: number | CommandOptions,
79+
): Promise<SpawnResult> {
80+
return await runCommandWithOutputEncoding(argv, optionsOrTimeout, true);
81+
}
82+
83+
async function runCommandWithOutputEncoding(
84+
argv: string[],
85+
optionsOrTimeout: number | CommandOptions,
86+
forceUtf8: boolean,
7187
): Promise<SpawnResult> {
7288
const options: CommandOptions =
7389
typeof optionsOrTimeout === "number" ? { timeoutMs: optionsOrTimeout } : optionsOrTimeout;
@@ -123,7 +139,7 @@ export async function runCommandWithTimeout(
123139
MAX_PRESERVED_PENDING_LINE_BYTES,
124140
);
125141
const maxPreservedOutputLines = Math.max(0, Math.floor(options.maxPreservedOutputLines ?? 16));
126-
const windowsEncoding = resolveWindowsConsoleEncoding();
142+
const windowsEncoding = forceUtf8 ? null : resolveWindowsConsoleEncoding();
127143
const cancelController = new AbortController();
128144
let termination: CommandTerminationReason | undefined;
129145
let childExitState: { code: number | null; signal: NodeJS.Signals | null } | undefined;
@@ -447,16 +463,20 @@ export async function runCommandWithTimeout(
447463
}
448464
}
449465

466+
const decodeCapturedOutput = (
467+
capture: CapturedOutputBuffers,
468+
captureMode: CommandOutputCaptureMode,
469+
): string => {
470+
const buffer = finalizeCapturedOutput(capture, captureMode, forceUtf8);
471+
return forceUtf8
472+
? buffer.toString("utf8")
473+
: decodeWindowsOutputBuffer({ buffer, windowsEncoding });
474+
};
475+
450476
return {
451477
pid: child.pid,
452-
stdout: decodeWindowsOutputBuffer({
453-
buffer: finalizeCapturedOutput(stdoutCapture, stdoutCaptureMode),
454-
windowsEncoding,
455-
}),
456-
stderr: decodeWindowsOutputBuffer({
457-
buffer: finalizeCapturedOutput(stderrCapture, stderrCaptureMode),
458-
windowsEncoding,
459-
}),
478+
stdout: decodeCapturedOutput(stdoutCapture, stdoutCaptureMode),
479+
stderr: decodeCapturedOutput(stderrCapture, stderrCaptureMode),
460480
stdoutTruncatedBytes: stdoutCapture.truncatedBytes || undefined,
461481
stderrTruncatedBytes: stderrCapture.truncatedBytes || undefined,
462482
preservedStdoutLines:

‎src/process/exec.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import { releaseChildProcessOutputAfterExit } from "./child-process.js";
1010
import { resolveMaxOutputBytes, type CommandOutputStream } from "./exec-output.js";
1111
import { runCommandWithTimeout } from "./exec-runner.js";
1212
import { COMMAND_PROCESS_TREE_KILL_GRACE_MS, spawnCommand } from "./exec-spawn.js";
13-
export { runCommandWithTimeout } from "./exec-runner.js";
13+
export { runCommandWithTimeout, runUtf8CommandWithTimeout } from "./exec-runner.js";
1414
export type { CommandOptions } from "./exec-runner.js";
1515
export { isPlainCommandExitFailure, resolveProcessExitCode } from "./exec-result.js";
1616
export type { SpawnResult } from "./exec-result.js";

‎src/process/exec.windows.integration.test.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import process from "node:process";
2+
import { describe, expect, it } from "vitest";
3+
import { runUtf8CommandWithTimeout } from "./exec.js";
4+
5+
describe("runUtf8CommandWithTimeout Windows integration", () => {
6+
it.runIf(process.platform === "win32")(
7+
"keeps truncated UTF-8 head output on a code point boundary",
8+
async () => {
9+
const result = await runUtf8CommandWithTimeout(
10+
[process.execPath, "-e", "process.stdout.write('a😀z'); process.stderr.write('b😀y')"],
11+
{
12+
maxOutputBytes: 3,
13+
outputCapture: "head",
14+
timeoutMs: 3_000,
15+
},
16+
);
17+
18+
expect(result.stdout).toBe("a");
19+
expect(result.stderr).toBe("b");
20+
expect(result.stdoutTruncatedBytes).toBe(5);
21+
expect(result.stderrTruncatedBytes).toBe(5);
22+
},
23+
);
24+
});

‎src/process/exec.windows.test.ts‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ function expectCmdWrappedInvocation(call: ExecaCall, commandFragment = "pnpm.cmd
126126
}
127127

128128
let runCommandWithTimeout: typeof import("./exec.js").runCommandWithTimeout;
129+
let runUtf8CommandWithTimeout: typeof import("./exec.js").runUtf8CommandWithTimeout;
129130
let runExec: typeof import("./exec.js").runExec;
130131
let spawnCommand: typeof import("./exec.js").spawnCommand;
131132
let getWindowsInstallRoots: typeof import("../infra/windows-install-roots.js").getWindowsInstallRoots;
@@ -160,7 +161,8 @@ describe("Windows command execution", () => {
160161
});
161162
({ getWindowsInstallRoots, getWindowsSystem32ExePath } =
162163
await import("../infra/windows-install-roots.js"));
163-
({ runCommandWithTimeout, runExec, spawnCommand } = await import("./exec.js"));
164+
({ runCommandWithTimeout, runExec, runUtf8CommandWithTimeout, spawnCommand } =
165+
await import("./exec.js"));
164166
});
165167

166168
afterAll(() => {
@@ -529,6 +531,48 @@ describe("Windows command execution", () => {
529531
});
530532
});
531533

534+
it("keeps truncated UTF-8 head output on a code point boundary", async () => {
535+
execaMock.mockImplementationOnce(() =>
536+
createMockSubprocess({
537+
stdoutChunks: [Buffer.from("a😀z", "utf8")],
538+
stderrChunks: [Buffer.from("b😀y", "utf8")],
539+
}),
540+
);
541+
await withMockedWindowsPlatform(async () => {
542+
await expect(
543+
runUtf8CommandWithTimeout(["node", "utf8-output.js"], {
544+
maxOutputBytes: 3,
545+
outputCapture: "head",
546+
timeoutMs: 1_000,
547+
}),
548+
).resolves.toMatchObject({
549+
stdout: "a",
550+
stderr: "b",
551+
stdoutTruncatedBytes: 5,
552+
stderrTruncatedBytes: 5,
553+
});
554+
expect(spawnSyncMock).not.toHaveBeenCalled();
555+
});
556+
});
557+
558+
it("preserves complete legacy-code-page characters in truncated head output", async () => {
559+
execaMock.mockImplementationOnce(() =>
560+
createMockSubprocess({ stdoutChunks: [Buffer.from([0x61, 0xb2, 0xe2, 0xca, 0xd4])] }),
561+
);
562+
await withMockedWindowsPlatform(async () => {
563+
await expect(
564+
runCommandWithTimeout(["node", "gbk-output.js"], {
565+
maxOutputBytes: 3,
566+
outputCapture: "head",
567+
timeoutMs: 1_000,
568+
}),
569+
).resolves.toMatchObject({
570+
stdout: "a测",
571+
stdoutTruncatedBytes: 2,
572+
});
573+
});
574+
});
575+
532576
it("decodes split GBK chunks after complete output capture", async () => {
533577
execaMock.mockImplementationOnce(() =>
534578
createMockSubprocess({

0 commit comments

Comments
 (0)