Skip to content

Commit 0bdd646

Browse files
wings1029steipete
wings1029
and
steipete
authored
fix(text): keep context tails UTF-16 safe (#102599)
Co-authored-by: Peter Steinberger <[email protected]>
1 parent 6621ead commit 0bdd646

6 files changed

Lines changed: 82 additions & 27 deletions

File tree

‎extensions/codex/src/app-server/context-engine-projection.test.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,17 @@ describe("projectContextEngineAssemblyForCodex", () => {
157157
expect(result.promptText.length).toBeLessThan(25_000);
158158
});
159159

160+
it("reports the exact text dropped when a text-part boundary crosses an emoji", () => {
161+
const prefix = "x".repeat(5_999);
162+
const result = projectContextEngineAssemblyForCodex({
163+
assembledMessages: [textMessage("assistant", `${prefix}😀tail`)],
164+
originalHistoryMessages: [],
165+
prompt: "next",
166+
});
167+
168+
expect(result.promptText).toContain(`[assistant]\n${prefix}\n[truncated 6 chars]`);
169+
});
170+
160171
it("keeps recent context when the rendered conversation overflows", () => {
161172
const result = projectContextEngineAssemblyForCodex({
162173
assembledMessages: [

‎extensions/codex/src/app-server/context-engine-projection.ts‎

Lines changed: 7 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
*/
55
import type { AgentMessage } from "openclaw/plugin-sdk/agent-harness-runtime";
66
import { redactSensitiveFieldValue, redactToolPayloadText } from "openclaw/plugin-sdk/logging-core";
7+
import { sliceUtf16Safe, truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
78

89
type CodexContextProjection = {
910
developerInstructionAddition?: string;
@@ -485,9 +486,11 @@ function resolveTextPartMaxChars(maxRenderedContextChars: number): number {
485486
}
486487

487488
function truncateText(text: string, maxChars: number): string {
488-
return text.length > maxChars
489-
? `${text.slice(0, maxChars)}\n[truncated ${text.length - maxChars} chars]`
490-
: text;
489+
if (text.length <= maxChars) {
490+
return text;
491+
}
492+
const truncated = truncateUtf16Safe(text, maxChars);
493+
return `${truncated}\n[truncated ${text.length - truncated.length} chars]`;
491494
}
492495

493496
function truncateOlderContext(text: string, maxChars: number): string {
@@ -507,20 +510,5 @@ function truncateOlderContext(text: string, maxChars: number): string {
507510
return marker.slice(0, maxChars);
508511
}
509512
tailChars = maxChars - marker.length;
510-
return `${marker}${sliceTailFromCodePointBoundary(text, tailChars).trimStart()}`;
511-
}
512-
513-
// Keep the kept tail at a code-point boundary so a UTF-16 surrogate pair is
514-
// never split at the cut: a tail start that lands on a low surrogate would
515-
// orphan it into U+FFFD, corrupting the first character. Dropping that unit
516-
// stays within maxChars (it only removes a char), so the bound still holds.
517-
function sliceTailFromCodePointBoundary(text: string, tailChars: number): string {
518-
let start = text.length - tailChars;
519-
if (start > 0 && start < text.length) {
520-
const code = text.charCodeAt(start);
521-
if (code >= 0xdc00 && code <= 0xdfff) {
522-
start += 1;
523-
}
524-
}
525-
return text.slice(start);
513+
return `${marker}${sliceUtf16Safe(text, -tailChars).trimStart()}`;
526514
}

‎src/agents/cli-runner/session-history.test.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -864,6 +864,18 @@ describe("buildCliSessionHistoryPrompt", () => {
864864
expect(prompt).not.toContain("x".repeat(80));
865865
});
866866

867+
it("keeps a whole code point when the retained history tail starts inside an emoji", () => {
868+
const prompt = buildCliSessionHistoryPrompt({
869+
messages: [{ role: "user", content: "prefix😀tail" }],
870+
prompt: "next",
871+
maxHistoryChars: 5,
872+
});
873+
874+
expect(prompt).toContain(
875+
"<conversation_history>\n[OpenClaw reseed history truncated; older turns dropped]\ntail\n</conversation_history>",
876+
);
877+
});
878+
867879
it("scales automatic reseed history caps from Claude context tiers", () => {
868880
expect(resolveAutoCliSessionReseedHistoryChars(0)).toBe(MAX_CLI_SESSION_RESEED_HISTORY_CHARS);
869881
expect(resolveAutoCliSessionReseedHistoryChars(32_000)).toBe(
@@ -980,6 +992,18 @@ describe("buildCliSessionHistoryPrompt", () => {
980992
expect(prompt).toContain("<next_user_message>\nnext ask\n</next_user_message>");
981993
});
982994

995+
it("keeps a whole code point at an oversize compaction-summary boundary", () => {
996+
const prompt = buildCliSessionHistoryPrompt({
997+
messages: [{ role: "compactionSummary", summary: `aa😀${"z".repeat(100)}` }],
998+
prompt: "next",
999+
maxHistoryChars: 80,
1000+
});
1001+
1002+
expect(prompt).toContain(
1003+
"<conversation_history>\n[OpenClaw reseed history truncated; older turns dropped]\nCompaction summary: aa\n</conversation_history>",
1004+
);
1005+
});
1006+
9831007
it("honors the cap when the summary block plus marker crosses it", () => {
9841008
// Edge case: `summaryRendered.length < maxHistoryChars` (the gate that
9851009
// routes to the oversize-summary branch is not taken) BUT

‎src/agents/cli-runner/session-history.ts‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
*/
55
import fsp from "node:fs/promises";
66
import path from "node:path";
7+
import { sliceUtf16Safe, truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
78
import {
89
resolveSessionFilePath,
910
resolveSessionFilePathOptions,
@@ -214,8 +215,8 @@ export function buildCliSessionHistoryPrompt(params: {
214215
0,
215216
maxHistoryChars - truncationMarker.length - separatorBudget - tailBudget,
216217
);
217-
const summaryTruncated = renderedSummary.slice(0, summaryBudget).trimEnd();
218-
const tailTruncated = tailBudget > 0 ? tailRaw.slice(-tailBudget).trimStart() : "";
218+
const summaryTruncated = truncateUtf16Safe(renderedSummary, summaryBudget).trimEnd();
219+
const tailTruncated = tailBudget > 0 ? sliceUtf16Safe(tailRaw, -tailBudget).trimStart() : "";
219220
return [truncationMarker, summaryTruncated, tailTruncated].filter(Boolean).join("\n");
220221
};
221222

@@ -242,7 +243,7 @@ export function buildCliSessionHistoryPrompt(params: {
242243
// reserved tail budget instead of being dropped wholesale.
243244
renderedHistory = renderTruncatedSummaryWithTail(summaryRendered);
244245
} else if (tailRaw.length > remainingBudget) {
245-
renderedHistory = `${summaryBlock}${truncationMarker}\n${tailRaw.slice(-remainingBudget).trimStart()}`;
246+
renderedHistory = `${summaryBlock}${truncationMarker}\n${sliceUtf16Safe(tailRaw, -remainingBudget).trimStart()}`;
246247
} else {
247248
renderedHistory = `${summaryBlock}${tailRaw}`;
248249
}
@@ -253,7 +254,7 @@ export function buildCliSessionHistoryPrompt(params: {
253254
// (older turns dropped, recent tail retained).
254255
renderedHistory =
255256
tailRaw.length > maxHistoryChars
256-
? `${truncationMarker}\n${tailRaw.slice(-maxHistoryChars).trimStart()}`
257+
? `${truncationMarker}\n${sliceUtf16Safe(tailRaw, -maxHistoryChars).trimStart()}`
257258
: tailRaw;
258259
}
259260

‎src/tui/tui-local-shell.test.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,35 @@ describe("createLocalShellRunner", () => {
165165
expect(harness.messages.some((m) => m.includes("FATAL"))).toBe(true);
166166
});
167167

168+
it("keeps a whole code point when the combined output tail starts inside an emoji", async () => {
169+
const stdout = new EventEmitter();
170+
const stderr = new EventEmitter();
171+
const spawnCommand = vi.fn(() => ({
172+
stdout,
173+
stderr,
174+
on: (event: string, callback: (...args: unknown[]) => void) => {
175+
if (event === "close") {
176+
setImmediate(() => {
177+
stdout.emit("data", Buffer.from("x😀"));
178+
stderr.emit("data", Buffer.from("tail"));
179+
callback(0, null);
180+
});
181+
}
182+
},
183+
}));
184+
const harness = createShellHarness({
185+
spawnCommand: spawnCommand as unknown as typeof import("node:child_process").spawn,
186+
maxOutputChars: 6,
187+
});
188+
189+
const run = harness.runLocalShellLine("!unicode");
190+
harness.getLastSelector()?.onSelect?.({ value: "yes", label: "Yes" });
191+
await run;
192+
193+
expect(harness.messages).toContain("[local] tail");
194+
expect(harness.messages.join("\n")).not.toMatch(/[\uD800-\uDFFF]/u);
195+
});
196+
168197
it("refuses to retarget local commands after the working directory is deleted", async () => {
169198
const harness = createShellHarness({ getCwd: () => undefined });
170199

‎src/tui/tui-local-shell.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Launches and manages the local shell process used by TUI local mode.
22
import { spawn } from "node:child_process";
33
import type { Component, OverlayHandle, SelectItem } from "@earendil-works/pi-tui";
4+
import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
45
import { tryProcessCwd } from "../infra/safe-cwd.js";
56
import { createSearchableSelectList } from "./components/selectors.js";
67

@@ -114,7 +115,7 @@ export function createLocalShellRunner(deps: LocalShellDeps) {
114115

115116
const appendWithCap = (text: string, chunk: string) => {
116117
const combined = text + chunk;
117-
return combined.length > maxChars ? combined.slice(-maxChars) : combined;
118+
return combined.length > maxChars ? sliceUtf16Safe(combined, -maxChars) : combined;
118119
};
119120

120121
await new Promise<void>((resolve) => {
@@ -143,9 +144,10 @@ export function createLocalShellRunner(deps: LocalShellDeps) {
143144
// Keep the tail (consistent with the streaming appendWithCap above) so a
144145
// large stdout cannot evict stderr: the failure reason (FATAL etc.) at the
145146
// end is what the operator needs most when output overflows the cap.
146-
const combined = (stdout + (stderr ? (stdout ? "\n" : "") + stderr : ""))
147-
.slice(-maxChars)
148-
.trimEnd();
147+
const combined = sliceUtf16Safe(
148+
stdout + (stderr ? (stdout ? "\n" : "") + stderr : ""),
149+
-maxChars,
150+
).trimEnd();
149151

150152
if (combined) {
151153
for (const lineLocal of combined.split("\n")) {

0 commit comments

Comments
 (0)