Skip to content

Commit e1bb413

Browse files
vincentkocZengWen-DT
vincentkoc
and
ZengWen-DT
committed
fix(core): narrow UTF-16 truncation repair
Co-authored-by: xialonglee <[email protected]> Co-authored-by: ZengWen-DT <[email protected]>
1 parent 8aeac0d commit e1bb413

8 files changed

Lines changed: 93 additions & 205 deletions

File tree

‎extensions/active-memory/index.test.ts‎

Lines changed: 53 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,27 @@ describe("active-memory plugin", () => {
290290
(result as { prependContext?: unknown } | undefined)?.prependContext,
291291
"expected prependContext",
292292
);
293+
const runRecallWithSummary = async (params: {
294+
prompt: string;
295+
summary: string;
296+
memoryText?: string;
297+
}): Promise<string> => {
298+
runEmbeddedAgent.mockImplementationOnce(async (runParams: { sessionFile: string }) => {
299+
await writeUsableMemoryTranscript(runParams.sessionFile, params.memoryText ?? params.summary);
300+
return { payloads: [{ text: params.summary }] };
301+
});
302+
return requirePrependContext(
303+
await hooks.before_prompt_build(
304+
{ prompt: params.prompt, messages: [] },
305+
{
306+
agentId: "main",
307+
trigger: "user",
308+
sessionKey: "agent:main:main",
309+
messageProvider: "webchat",
310+
},
311+
),
312+
);
313+
};
293314
const expectPrependContextContains = (result: unknown, text: string) => {
294315
expect(requirePrependContext(result)).toContain(text);
295316
};
@@ -5335,34 +5356,45 @@ describe("active-memory plugin", () => {
53355356
maxSummaryChars: 40,
53365357
};
53375358
plugin.register(api as unknown as OpenClawPluginApi);
5338-
runEmbeddedAgent.mockImplementationOnce(async (params: { sessionFile: string }) => {
5339-
await writeUsableMemoryTranscript(params.sessionFile, "alpha beta gamma");
5340-
return {
5341-
payloads: [
5342-
{
5343-
text: "alpha beta gamma delta epsilon zetalongword",
5344-
},
5345-
],
5346-
};
5359+
const prependContext = await runRecallWithSummary({
5360+
prompt: "what wings should i order? word-boundary-truncation-40",
5361+
summary: "alpha beta gamma delta epsilon zetalongword",
5362+
memoryText: "alpha beta gamma",
53475363
});
5348-
5349-
const result = await hooks.before_prompt_build(
5350-
{ prompt: "what wings should i order? word-boundary-truncation-40", messages: [] },
5351-
{
5352-
agentId: "main",
5353-
trigger: "user",
5354-
sessionKey: "agent:main:main",
5355-
messageProvider: "webchat",
5356-
},
5357-
);
5358-
5359-
const prependContext = requirePrependContext(result);
53605364
expect(prependContext).toContain("alpha beta gamma");
53615365
expect(prependContext).toContain("alpha beta gamma delta epsilon…");
53625366
expect(prependContext).not.toContain("zetalo");
53635367
expect(prependContext).not.toContain("zetalongword");
53645368
});
53655369

5370+
it.each([
5371+
{
5372+
name: "split surrogate",
5373+
summary: `${"a".repeat(38)}🎉TAILWORD`,
5374+
expected: `${"a".repeat(38)}…`,
5375+
},
5376+
{
5377+
name: "whitespace before a split surrogate",
5378+
summary: `alpha beta ${"c".repeat(26)} 🎉TAILWORD`,
5379+
expected: `alpha beta ${"c".repeat(26)}…`,
5380+
},
5381+
])("keeps $name truncation UTF-16 safe", async ({ name, summary, expected }) => {
5382+
api.pluginConfig = {
5383+
agents: ["main"],
5384+
maxSummaryChars: 40,
5385+
};
5386+
plugin.register(api as unknown as OpenClawPluginApi);
5387+
5388+
const prependContext = await runRecallWithSummary({
5389+
prompt: `recall summary boundary: ${name}`,
5390+
summary,
5391+
memoryText: expected,
5392+
});
5393+
5394+
expect(prependContext).toContain(expected);
5395+
expect(prependContext).not.toContain("TAILWORD");
5396+
});
5397+
53665398
it("asks recall subagents to mark mutable operational facts stale unless source status is current", async () => {
53675399
await hooks.before_prompt_build(
53685400
{ prompt: "is autonomous pickup running?", messages: [] },

‎extensions/active-memory/index.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import {
3737
uniqueStrings,
3838
} from "openclaw/plugin-sdk/string-coerce-runtime";
3939
import { tempWorkspace, resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
40+
import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
4041

4142
const DEFAULT_TIMEOUT_MS = 15_000;
4243
const DEFAULT_AGENT_ID = "main";
@@ -2570,15 +2571,16 @@ function truncateSummary(summary: string, maxSummaryChars: number): string {
25702571
return ellipsis.slice(0, Math.max(0, maxSummaryChars));
25712572
}
25722573
const contentMaxChars = maxSummaryChars - ellipsis.length;
2573-
const bounded = trimmed.slice(0, contentMaxChars).trimEnd();
2574+
const rawBounded = trimmed.slice(0, contentMaxChars).trimEnd();
2575+
const bounded = truncateUtf16Safe(trimmed, contentMaxChars).trimEnd();
25742576
const nextChar = trimmed.charAt(contentMaxChars);
25752577
if (!nextChar || /\s/.test(nextChar)) {
25762578
return `${bounded}${ellipsis}`;
25772579
}
25782580

2579-
const lastBoundary = bounded.search(/\s\S*$/);
2581+
const lastBoundary = rawBounded.search(/\s\S*$/);
25802582
if (lastBoundary > 0) {
2581-
return `${bounded.slice(0, lastBoundary).trimEnd()}${ellipsis}`;
2583+
return `${truncateUtf16Safe(trimmed, lastBoundary).trimEnd()}${ellipsis}`;
25822584
}
25832585

25842586
return `${bounded}${ellipsis}`;

‎src/agents/acp-spawn-parent-stream.test.ts‎

Lines changed: 31 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -100,26 +100,6 @@ function firstMockCall(
100100
return call;
101101
}
102102

103-
function hasLoneSurrogate(text: string): boolean {
104-
for (let i = 0; i < text.length; i++) {
105-
const code = text.charCodeAt(i);
106-
if (code >= 0xdc00 && code <= 0xdfff) {
107-
return true;
108-
}
109-
if (code >= 0xd800 && code <= 0xdbff) {
110-
if (i + 1 >= text.length) {
111-
return true;
112-
}
113-
const next = text.charCodeAt(i + 1);
114-
if (next < 0xdc00 || next > 0xdfff) {
115-
return true;
116-
}
117-
i++;
118-
}
119-
}
120-
return false;
121-
}
122-
123103
describe("startAcpSpawnParentStreamRelay", () => {
124104
beforeAll(async () => {
125105
({ emitAgentEvent } = await import("../infra/agent-events.js"));
@@ -1426,6 +1406,37 @@ describe("startAcpSpawnParentStreamRelay", () => {
14261406
relay.dispose();
14271407
});
14281408

1409+
it.each([
1410+
{
1411+
name: "preview cutoff",
1412+
delta: `${"a".repeat(218)}😀tail`,
1413+
expected: `${"a".repeat(218)}…`,
1414+
},
1415+
{
1416+
name: "retained buffer start",
1417+
delta: `😀${"b".repeat(3_999)}`,
1418+
expected: `${"b".repeat(219)}…`,
1419+
},
1420+
])("keeps $name on UTF-16 boundaries", ({ delta, expected }) => {
1421+
const relay = startAcpSpawnParentStreamRelay({
1422+
runId: "run-utf16-safe",
1423+
parentSessionKey: "agent:main:main",
1424+
childSessionKey: "agent:codex:acp:utf16-safe",
1425+
agentId: "codex",
1426+
streamFlushMs: 0,
1427+
noOutputNoticeMs: 120_000,
1428+
});
1429+
1430+
emitAgentEvent({
1431+
runId: "run-utf16-safe",
1432+
stream: "assistant",
1433+
data: { delta },
1434+
});
1435+
1436+
expect(collectedTexts()[1]).toBe(`codex: ${expected}`);
1437+
relay.dispose();
1438+
});
1439+
14291440
it("resolves ACP spawn stream log path from session metadata", () => {
14301441
readAcpSessionEntryMock.mockReturnValue({
14311442
storePath: "/tmp/openclaw/agents/codex/sessions/sessions.json",
@@ -1455,36 +1466,4 @@ describe("startAcpSpawnParentStreamRelay", () => {
14551466
expect(entry.sessionId).toBe("sess-123");
14561467
expect(options.storePath).toBe("/tmp/openclaw/agents/codex/sessions/sessions.json");
14571468
});
1458-
1459-
it("does not split surrogate pairs when truncating streamed output", () => {
1460-
const relay = startAcpSpawnParentStreamRelay({
1461-
runId: "run-utf16-safe",
1462-
parentSessionKey: "agent:main:main",
1463-
childSessionKey: "agent:codex:acp:utf16-safe",
1464-
agentId: "codex",
1465-
streamFlushMs: 0,
1466-
noOutputNoticeMs: 120_000,
1467-
});
1468-
1469-
// 218 "a" + "😀" + "moretext" = 228 chars. "😀" occupies positions 218-219.
1470-
// truncate(value, STREAM_SNIPPET_MAX_CHARS=220): old slice(0, 219) keeps
1471-
// positions 0-218 = "a".repeat(218) + high surrogate (lone!).
1472-
// truncateUtf16Safe(value, 219) backs off to 218.
1473-
const delta = `${"a".repeat(218)}😀moretext`;
1474-
1475-
emitAgentEvent({
1476-
runId: "run-utf16-safe",
1477-
stream: "assistant",
1478-
data: { delta },
1479-
});
1480-
1481-
const texts = collectedTexts();
1482-
// Skip the start notice, check only progress and completion events.
1483-
const relayTexts = texts.filter((t) => t.includes("codex:") || t.includes("codex "));
1484-
for (const text of relayTexts) {
1485-
expect(hasLoneSurrogate(text)).toBe(false);
1486-
}
1487-
1488-
relay.dispose();
1489-
});
14901469
});

‎src/agents/acp-spawn-parent-stream.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { mkdir } from "node:fs/promises";
33
import path from "node:path";
44
import { asFiniteNumber } from "@openclaw/normalization-core/number-coercion";
55
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
6+
import { sliceUtf16Safe, truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
67
import { readAcpSessionEntry } from "../acp/runtime/session-meta.js";
78
import {
89
isAcpTagVisible,
@@ -28,7 +29,6 @@ import { resolveNormalizedAccountEntry } from "../routing/account-lookup.js";
2829
import { normalizeAccountId } from "../routing/session-key.js";
2930
import { normalizeAssistantPhase } from "../shared/chat-message-content.js";
3031
import { recordTaskRunProgressByRunId } from "../tasks/detached-task-runtime.js";
31-
import { truncateUtf16Safe } from "../utils.js";
3232
import type { DeliveryContext } from "../utils/delivery-context.types.js";
3333

3434
const DEFAULT_STREAM_FLUSH_MS = 2_500;
@@ -504,7 +504,7 @@ export function startAcpSpawnParentStreamRelay(params: {
504504
pendingProgressKind = kind;
505505
pendingText += delta;
506506
if (pendingText.length > STREAM_BUFFER_MAX_CHARS) {
507-
pendingText = pendingText.slice(-STREAM_BUFFER_MAX_CHARS);
507+
pendingText = sliceUtf16Safe(pendingText, -STREAM_BUFFER_MAX_CHARS);
508508
}
509509
if (pendingText.length >= STREAM_SNIPPET_MAX_CHARS || delta.includes("\n\n")) {
510510
flushPending();

‎src/agents/harness/native-hook-relay.test.ts‎

Lines changed: 0 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -33,26 +33,6 @@ afterEach(() => {
3333
testing.clearNativeHookRelaysForTests();
3434
});
3535

36-
function hasLoneSurrogate(text: string): boolean {
37-
for (let i = 0; i < text.length; i++) {
38-
const code = text.charCodeAt(i);
39-
if (code >= 0xdc00 && code <= 0xdfff) {
40-
return true;
41-
}
42-
if (code >= 0xd800 && code <= 0xdbff) {
43-
if (i + 1 >= text.length) {
44-
return true;
45-
}
46-
const next = text.charCodeAt(i + 1);
47-
if (next < 0xdc00 || next > 0xdfff) {
48-
return true;
49-
}
50-
i++;
51-
}
52-
}
53-
return false;
54-
}
55-
5636
function isRecord(value: unknown): value is Record<string, unknown> {
5737
return typeof value === "object" && value !== null && !Array.isArray(value);
5838
}
@@ -3251,37 +3231,3 @@ describe("native hook relay command builder", () => {
32513231
);
32523232
});
32533233
});
3254-
3255-
describe("UTF-16 safe truncation in permission approval display", () => {
3256-
it("does not split surrogate pairs when truncating approval description command text", () => {
3257-
// 236 "c" + "😀" + "tail" = 242 chars. The emoji at positions 236-237
3258-
// crosses the 240-char truncateText() boundary (maxLength - 3 = 237).
3259-
// Old raw slice(0, 237) would keep the lone high surrogate at position 236;
3260-
// truncateUtf16Safe backs off to 236.
3261-
const command = `${"c".repeat(236)}😀tail`;
3262-
const description = testing.formatPermissionApprovalDescriptionForTests({
3263-
provider: "codex",
3264-
sessionId: "sess-utf16",
3265-
runId: "run-utf16",
3266-
toolName: "exec",
3267-
toolInput: { command },
3268-
});
3269-
3270-
expect(hasLoneSurrogate(description)).toBe(false);
3271-
});
3272-
3273-
it("does not split surrogate pairs when truncating approval description Cwd text", () => {
3274-
// Same pattern for the Cwd field (240-char limit via truncateText).
3275-
const cwd = `${"/".repeat(236)}😀tail`;
3276-
const description = testing.formatPermissionApprovalDescriptionForTests({
3277-
provider: "codex",
3278-
sessionId: "sess-utf16",
3279-
runId: "run-utf16",
3280-
toolName: "exec",
3281-
cwd,
3282-
toolInput: {},
3283-
});
3284-
3285-
expect(hasLoneSurrogate(description)).toBe(false);
3286-
});
3287-
});

‎src/agents/harness/native-hook-relay.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ import { privateFileStoreSync } from "../../infra/private-file-store.js";
3131
import { createSubsystemLogger } from "../../logging/subsystem.js";
3232
import { hasGlobalHooks } from "../../plugins/hook-runner-global.js";
3333
import { PluginApprovalResolutions } from "../../plugins/types.js";
34-
import { truncateUtf16Safe } from "../../utils.js";
3534
import {
3635
cancelDeferredPluginToolApproval,
3736
hasBeforeToolCallPolicy,
@@ -2135,7 +2134,7 @@ function truncateText(value: string, maxLength: number): string {
21352134
if (value.length <= maxLength) {
21362135
return value;
21372136
}
2138-
return `${truncateUtf16Safe(value, Math.max(0, maxLength - 3))}...`;
2137+
return `${value.slice(0, Math.max(0, maxLength - 3))}...`;
21392138
}
21402139

21412140
function resolveOpenClawCliExecutable(): string {

0 commit comments

Comments
 (0)