Skip to content

Commit b311a3c

Browse files
wsyjh8wsyjh8
committed
fix(daemon): mark code-page cmd launchers with their encoding for deterministic readback
Prepend an ASCII '@Rem openclaw-launcher-encoding=<label>' line to code-page .cmd launchers and decode by that marker instead of sniffing UTF-8. Some GBK byte sequences are valid UTF-8 (隆 = C2 A1 reads as ¡), so the old sniff silently corrupted readback and rejected valid paths; the marker makes decode deterministic and drops the code-page probe (a PowerShell spawn) from the frequent readScheduledTaskCommand poll path. Also fix the representability guard for euc-kr: Node ICU decodes euc-kr as KS X 1001 only, but Windows code page 949 is cp949/UHC, so the TextDecoder cross-check false-rejected ~8,800 UHC extension syllables (똠 = 8C 63) that iconv encodes and cmd.exe reads fine. Verify euc-kr via iconv's own cp949 round-trip; keep TextDecoder for the other five labels.
1 parent 7627f95 commit b311a3c

3 files changed

Lines changed: 135 additions & 58 deletions

File tree

‎src/daemon/launcher-encoding.test.ts‎

Lines changed: 64 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ vi.mock("../infra/windows-encoding.js", async () => {
1717

1818
const CJK_SCRIPT_PATH = "C:\\Users\\苗振\\.openclaw\\gateway.cmd";
1919
const REPLACEMENT_CHAR = String.fromCharCode(0xfffd);
20+
const GBK_MARKER = "@rem openclaw-launcher-encoding=gbk\r\n";
21+
const EUC_KR_MARKER = "@rem openclaw-launcher-encoding=euc-kr\r\n";
2022

2123
beforeEach(() => {
2224
resolveWindowsSystemEncodingMock.mockReset();
@@ -51,7 +53,7 @@ describe("encodeWindowsLauncherScript", () => {
5153
expect(encoded.equals(Buffer.from(content, "utf8"))).toBe(true);
5254
});
5355

54-
it("encodes non-ASCII cmd scripts with the CJK system code page", () => {
56+
it("encodes non-ASCII cmd scripts with a marker line plus CJK code page bytes", () => {
5557
const content = `@echo off\r\ncd /d "C:\\Users\\苗振\\.openclaw"\r\nnode gateway.js\r\n`;
5658
const encoded = encodeWindowsLauncherScript({
5759
format: "cmd",
@@ -60,8 +62,48 @@ describe("encodeWindowsLauncherScript", () => {
6062
});
6163

6264
expect(encoded.equals(Buffer.from(content, "utf8"))).toBe(false);
63-
expect(encoded.equals(iconv.encode(content, "gbk"))).toBe(true);
64-
expect(decodeWindowsLauncherScript({ buffer: encoded, windowsEncoding: "gbk" })).toBe(content);
65+
expect(encoded.equals(iconv.encode(GBK_MARKER + content, "gbk"))).toBe(true);
66+
expect(decodeWindowsLauncherScript({ buffer: encoded })).toBe(content);
67+
});
68+
69+
it("round-trips cmd content whose GBK bytes are also valid UTF-8 (隆 -> C2 A1 -> ¡)", () => {
70+
// Verify the trap on this iconv build: valid UTF-8, wrong string, no U+FFFD.
71+
const collisionBytes = iconv.encode("隆", "gbk");
72+
expect(collisionBytes.toString("utf8")).not.toBe("隆");
73+
expect(collisionBytes.toString("utf8")).not.toContain(REPLACEMENT_CHAR);
74+
75+
const content = `@echo off\r\ncd /d "C:\\Users\\隆\\.openclaw"\r\nnode gateway.js\r\n`;
76+
const encoded = encodeWindowsLauncherScript({
77+
format: "cmd",
78+
content,
79+
windowsEncoding: "gbk",
80+
});
81+
82+
expect(encoded.equals(iconv.encode(GBK_MARKER + content, "gbk"))).toBe(true);
83+
// Locks the regression: a raw UTF-8 readback of these bytes decodes
84+
// cleanly (no replacement char) yet corrupts the path — the pre-marker bug.
85+
expect(encoded.toString("utf8")).not.toContain(REPLACEMENT_CHAR);
86+
expect(encoded.toString("utf8")).not.toContain("隆");
87+
expect(decodeWindowsLauncherScript({ buffer: encoded })).toBe(content);
88+
});
89+
90+
it("encodes cp949 extension syllables that Node ICU's euc-kr decoder rejects", () => {
91+
// Windows code page 949 is cp949/UHC; "똠" (8C 63) is a UHC extension syllable
92+
// iconv encodes and round-trips, but new TextDecoder("euc-kr") cannot decode
93+
// (KS X 1001 only). The guard must verify euc-kr with iconv, not ICU.
94+
const extensionBytes = iconv.encode("똠", "euc-kr");
95+
expect(iconv.decode(extensionBytes, "euc-kr")).toBe("똠");
96+
expect(new TextDecoder("euc-kr").decode(extensionBytes)).not.toBe("똠");
97+
98+
const content = `@echo off\r\ncd /d "C:\\Users\\똠이\\.openclaw"\r\nnode gateway.js\r\n`;
99+
const encoded = encodeWindowsLauncherScript({
100+
format: "cmd",
101+
content,
102+
windowsEncoding: "euc-kr",
103+
});
104+
105+
expect(encoded.equals(iconv.encode(EUC_KR_MARKER + content, "euc-kr"))).toBe(true);
106+
expect(decodeWindowsLauncherScript({ buffer: encoded })).toBe(content);
65107
});
66108

67109
it("falls back to UTF-8 when no system code page is available", () => {
@@ -91,7 +133,7 @@ describe("encodeWindowsLauncherScript", () => {
91133
const content = `@echo off\r\ncd /d "C:\\Users\\苗振"\r\n`;
92134
const encoded = encodeWindowsLauncherScript({ format: "cmd", content });
93135

94-
expect(encoded.equals(iconv.encode(content, "gbk"))).toBe(true);
136+
expect(encoded.equals(iconv.encode(GBK_MARKER + content, "gbk"))).toBe(true);
95137
});
96138

97139
it("fails the install instead of writing unrepresentable cmd content", () => {
@@ -111,29 +153,33 @@ describe("decodeWindowsLauncherScript", () => {
111153
expect(decodeWindowsLauncherScript({ buffer })).toBe(content);
112154
});
113155

114-
it("prefers valid UTF-8 over the system code page for legacy scripts", () => {
156+
it("decodes unmarked legacy UTF-8 scripts with CJK paths", () => {
115157
const content = `@echo off\r\ncd /d "C:\\Users\\苗振\\.openclaw"\r\nnode gateway.js\r\n`;
116158
const buffer = Buffer.from(content, "utf8");
117159

118-
expect(decodeWindowsLauncherScript({ buffer, windowsEncoding: "gbk" })).toBe(content);
160+
expect(decodeWindowsLauncherScript({ buffer })).toBe(content);
161+
});
162+
163+
it("decodes marked code-page scripts with the recorded encoding", () => {
164+
const content = "@echo off\r\nrem 你好\r\n";
165+
const buffer = iconv.encode(GBK_MARKER + content, "gbk");
166+
167+
expect(decodeWindowsLauncherScript({ buffer })).toBe(content);
119168
});
120169

121-
it("decodes ANSI scripts with the system code page", () => {
122-
// GBK bytes for 你好, mirroring src/infra/windows-encoding.test.ts fixtures.
123-
const buffer = Buffer.concat([
124-
Buffer.from("@echo off\r\nrem ", "utf8"),
125-
Buffer.from([0xc4, 0xe3, 0xba, 0xc3]),
126-
]);
170+
it("falls back to UTF-8 for marker labels iconv cannot decode", () => {
171+
const content = "@rem openclaw-launcher-encoding=bogus\r\nnode gateway.js\r\n";
172+
const buffer = Buffer.from(content, "utf8");
127173

128-
expect(decodeWindowsLauncherScript({ buffer, windowsEncoding: "gbk" })).toBe(
129-
"@echo off\r\nrem 你好",
130-
);
174+
expect(decodeWindowsLauncherScript({ buffer })).toBe(content);
131175
});
132176

133-
it("degrades to UTF-8 replacement output when no code page is available", () => {
177+
it("degrades unmarked non-UTF-8 bytes to UTF-8 replacement output", () => {
178+
// Unmarked code-page files were never produced by a shipped release (the
179+
// code-page writer ships together with the marker), so deterministic UTF-8
180+
// is the correct total behavior for everything without a marker.
134181
const buffer = Buffer.from([0xc4, 0xe3, 0xba, 0xc3]);
135182

136-
const decoded = decodeWindowsLauncherScript({ buffer, windowsEncoding: null });
137-
expect(decoded).toContain(REPLACEMENT_CHAR);
183+
expect(decodeWindowsLauncherScript({ buffer })).toContain(REPLACEMENT_CHAR);
138184
});
139185
});

‎src/daemon/launcher-encoding.ts‎

Lines changed: 44 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
/** Encodes and decodes generated Windows launcher scripts (gateway.cmd / gateway.vbs). */
22
import iconv from "iconv-lite";
3-
import {
4-
decodeWindowsTextFileBuffer,
5-
resolveWindowsSystemEncoding,
6-
} from "../infra/windows-encoding.js";
3+
import { resolveWindowsSystemEncoding } from "../infra/windows-encoding.js";
74

85
type WindowsLauncherScriptFormat = "cmd" | "vbs";
96

@@ -22,6 +19,13 @@ const CMD_CONSOLE_SAFE_ENCODINGS = new Set([
2219
"windows-874",
2320
]);
2421

22+
// Code-page cmd launchers record their encoding in this ASCII comment line so
23+
// readback never has to guess: some code-page byte sequences are also valid
24+
// UTF-8 (GBK "隆" is C2 A1, which UTF-8 reads as "¡"), so content sniffing
25+
// silently corrupts paths.
26+
const LAUNCHER_ENCODING_MARKER_PREFIX = "@rem openclaw-launcher-encoding=";
27+
const LAUNCHER_ENCODING_MARKER_RE = /^@rem openclaw-launcher-encoding=(\S+)\s*$/;
28+
2529
function isAsciiOnly(value: string): boolean {
2630
for (let index = 0; index < value.length; index += 1) {
2731
if (value.charCodeAt(index) > 0x7f) {
@@ -55,39 +59,50 @@ export function encodeWindowsLauncherScript(params: {
5559
if (!encoding || !CMD_CONSOLE_SAFE_ENCODINGS.has(encoding) || !iconv.encodingExists(encoding)) {
5660
return Buffer.from(params.content, "utf8");
5761
}
58-
const encoded = iconv.encode(params.content, encoding);
62+
// Generated launcher scripts are CRLF-terminated throughout; the marker is
63+
// ASCII, so it encodes byte-identically in every safe code page.
64+
const marked = `${LAUNCHER_ENCODING_MARKER_PREFIX}${encoding}\r\n${params.content}`;
65+
const encoded = iconv.encode(marked, encoding);
5966
// iconv-lite substitutes "?" for unmappable characters, which would silently
60-
// corrupt paths; verify with the same decoder the read path uses and fail the
61-
// install before any launcher file is written.
62-
if (new TextDecoder(encoding).decode(encoded) !== params.content) {
67+
// corrupt paths; verify the round-trip and fail the install before any
68+
// launcher file is written. Node ICU's euc-kr decoder is KS X 1001 only, but
69+
// Windows code page 949 is cp949/UHC, so ICU false-rejects the ~8,800 UHC
70+
// extension syllables cmd.exe reads fine — verify euc-kr with iconv's own
71+
// cp949 decode instead. The other five labels match Windows in ICU, which
72+
// also flags best-fit hazards (shift_jis ¥ -> 0x5C) iconv's decode would miss.
73+
const decoded =
74+
encoding === "euc-kr"
75+
? iconv.decode(encoded, encoding)
76+
: new TextDecoder(encoding).decode(encoded);
77+
if (decoded !== marked) {
6378
throw new Error(
6479
`Windows ${params.format} launcher script contains characters that cannot be represented in the Windows system code page (${encoding}); cmd.exe would misread the script. Remove those characters or switch Windows to UTF-8 (code page 65001).`,
6580
);
6681
}
6782
return encoded;
6883
}
6984

70-
/** Decodes launcher scripts written by any OpenClaw version (UTF-16 LE BOM, UTF-8, or ANSI). */
71-
export function decodeWindowsLauncherScript(params: {
72-
buffer: Buffer;
73-
windowsEncoding?: string | null;
74-
}): string {
75-
if (params.buffer.length >= 2 && params.buffer[0] === 0xff && params.buffer[1] === 0xfe) {
76-
return params.buffer.subarray(2).toString("utf16le");
85+
/** Decodes launcher scripts written by any OpenClaw version (UTF-16 LE BOM, marked code page, or UTF-8). */
86+
export function decodeWindowsLauncherScript(params: { buffer: Buffer }): string {
87+
const { buffer } = params;
88+
if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) {
89+
return buffer.subarray(2).toString("utf16le");
7790
}
78-
// Valid UTF-8 first: covers ASCII and pre-fix UTF-8 installs without paying
79-
// for code page detection on frequent readScheduledTaskCommand polls.
80-
const utf8 = params.buffer.toString("utf8");
81-
if (!utf8.includes("\uFFFD")) {
82-
return utf8;
91+
// The marker line is pure ASCII and no CMD_CONSOLE_SAFE_ENCODINGS multibyte
92+
// sequence contains 0x0A, so the first newline in a marked file is exactly
93+
// the marker terminator. latin1 (not "ascii", which masks the high bit and
94+
// could alias garbage bytes into the prefix) keeps the byte-level read exact.
95+
const newlineIndex = buffer.indexOf(0x0a);
96+
if (newlineIndex !== -1) {
97+
const marker = LAUNCHER_ENCODING_MARKER_RE.exec(
98+
buffer.subarray(0, newlineIndex).toString("latin1"),
99+
);
100+
// encodingExists guards hand-edited marker labels so a bad label degrades
101+
// to the UTF-8 fallback instead of iconv.decode throwing mid-poll.
102+
if (marker?.[1] && iconv.encodingExists(marker[1])) {
103+
return iconv.decode(buffer.subarray(newlineIndex + 1), marker[1]);
104+
}
83105
}
84-
const encoding =
85-
params.windowsEncoding !== undefined ? params.windowsEncoding : resolveWindowsSystemEncoding();
86-
// Launcher files are Windows artifacts no matter which host runs this code,
87-
// so pin the platform instead of letting the decoder no-op off-Windows.
88-
return decodeWindowsTextFileBuffer({
89-
buffer: params.buffer,
90-
platform: "win32",
91-
windowsEncoding: encoding ?? "utf-8",
92-
});
106+
// No marker: ASCII scripts and pre-marker legacy UTF-8 installs.
107+
return buffer.toString("utf8");
93108
}

‎src/daemon/schtasks.test.ts‎

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
import fs from "node:fs/promises";
33
import os from "node:os";
44
import path from "node:path";
5-
import iconv from "iconv-lite";
65
import { beforeEach, describe, expect, it, vi } from "vitest";
6+
import { encodeWindowsLauncherScript } from "./launcher-encoding.js";
77
import {
88
readScheduledTaskCommand,
99
readScheduledTaskRuntime,
@@ -18,14 +18,6 @@ vi.mock("./schtasks-exec.js", () => ({
1818
execSchtasks: async () => schtasksResponses.shift() ?? { code: 0, stdout: "", stderr: "" },
1919
}));
2020

21-
// Pin code page detection to GBK so ANSI launcher decoding stays host-independent.
22-
vi.mock("../infra/windows-encoding.js", async () => {
23-
const actual = await vi.importActual<typeof import("../infra/windows-encoding.js")>(
24-
"../infra/windows-encoding.js",
25-
);
26-
return { ...actual, resolveWindowsSystemEncoding: () => "gbk" };
27-
});
28-
2921
beforeEach(() => {
3022
schtasksResponses.length = 0;
3123
});
@@ -229,7 +221,12 @@ describe("readScheduledTaskCommand", () => {
229221
await fs.writeFile(
230222
scriptPath,
231223
options.scriptEncoding === "gbk"
232-
? iconv.encode(script, "gbk")
224+
? // Production bytes for a code-page install: marker line + GBK body.
225+
encodeWindowsLauncherScript({
226+
format: "cmd",
227+
content: script,
228+
windowsEncoding: "gbk",
229+
})
233230
: Buffer.from(script, "utf8"),
234231
);
235232
}
@@ -271,7 +268,7 @@ describe("readScheduledTaskCommand", () => {
271268
);
272269
});
273270

274-
it("reads ANSI scripts with CJK paths under a CJK code page (#107416)", async () => {
271+
it("reads marked ANSI scripts with CJK paths under a CJK code page (#107416)", async () => {
275272
await withScheduledTaskScript(
276273
{
277274
scriptLines: ["@echo off", 'cd /d "C:\\Users\\苗振\\.openclaw"', "node gateway.js"],
@@ -288,6 +285,25 @@ describe("readScheduledTaskCommand", () => {
288285
);
289286
});
290287

288+
it("reads back GBK launchers whose bytes are also valid UTF-8 (隆) without corruption", async () => {
289+
// GBK "隆" is C2 A1, which UTF-8 accepts as "¡"; the marker keeps readback
290+
// from sniffing these bytes as UTF-8 and parsing a corrupted path.
291+
await withScheduledTaskScript(
292+
{
293+
scriptLines: ["@echo off", 'cd /d "C:\\Users\\隆\\.openclaw"', "node gateway.js"],
294+
scriptEncoding: "gbk",
295+
},
296+
async (env) => {
297+
const result = await readScheduledTaskCommand(env);
298+
expect(result).toEqual({
299+
programArguments: ["node", "gateway.js"],
300+
workingDirectory: "C:\\Users\\隆\\.openclaw",
301+
sourcePath: resolveTaskScriptPath(env),
302+
});
303+
},
304+
);
305+
});
306+
291307
it("returns null when script does not exist", async () => {
292308
await withScheduledTaskScript({}, async (env) => {
293309
const result = await readScheduledTaskCommand(env);

0 commit comments

Comments
 (0)