fix(codex,telegram,tui,cli): keep context projection, raw update, TUI shell, and session history truncation UTF-16 safe

wings1029 · wings1029 · commit d7104bdbf4ba · 2026-07-09T17:14:59.000+08:00
diff --git a/extensions/codex/src/app-server/context-engine-projection.test.ts b/extensions/codex/src/app-server/context-engine-projection.test.ts
@@ -421,3 +421,24 @@ describe("projectContextEngineAssemblyForCodex", () => {
     );
   });
 });
+
+describe("truncateText", () => {
+  it("returns the original text when it fits within maxChars", async () => {
+    const { truncateText } = await import("./context-engine-projection.js");
+    expect(truncateText("hello", 100)).toBe("hello");
+  });
+
+  it("truncates and reports the number of dropped characters", async () => {
+    const { truncateText } = await import("./context-engine-projection.js");
+    const result = truncateText("x".repeat(200), 100);
+    expect(result).toContain("[truncated");
+    expect(result).toContain("chars]");
+  });
+
+  it("does not split a surrogate pair at the truncation boundary", async () => {
+    const { truncateText } = await import("./context-engine-projection.js");
+    const input = `aa🚀${"b".repeat(200)}`;
+    const result = truncateText(input, 80);
+    expect(result).not.toContain("�");
+  });
+});
diff --git a/extensions/codex/src/app-server/context-engine-projection.ts b/extensions/codex/src/app-server/context-engine-projection.ts
@@ -1,3 +1,4 @@
+import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
 /**
  * Projects OpenClaw context-engine assemblies into Codex prompt text while
  * preserving safety boundaries and redacting tool payloads.
@@ -484,10 +485,13 @@ function resolveTextPartMaxChars(maxRenderedContextChars: number): number {
   );
 }
 
-function truncateText(text: string, maxChars: number): string {
-  return text.length > maxChars
-    ? `${text.slice(0, maxChars)}\n[truncated ${text.length - maxChars} chars]`
-    : text;
+export function truncateText(text: string, maxChars: number): string {
+  if (text.length <= maxChars) {
+    return text;
+  }
+  const sliced = truncateUtf16Safe(text, maxChars);
+  const truncatedChars = text.length - sliced.length;
+  return `${sliced}\n[truncated ${truncatedChars} chars]`;
 }
 
 function truncateOlderContext(text: string, maxChars: number): string {
diff --git a/extensions/telegram/src/bot-core.ts b/extensions/telegram/src/bot-core.ts
@@ -1,3 +1,4 @@
+import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
 // Telegram plugin module implements bot core behavior.
 import {
   resolveChannelGroupPolicy,
@@ -248,7 +249,9 @@ export function createTelegramBotCore(
       try {
         const raw = stringifyTelegramRawUpdateForLog(ctx.update);
         const preview =
-          raw.length > MAX_RAW_UPDATE_CHARS ? `${raw.slice(0, MAX_RAW_UPDATE_CHARS)}...` : raw;
+          raw.length > MAX_RAW_UPDATE_CHARS
+            ? `${truncateUtf16Safe(raw, MAX_RAW_UPDATE_CHARS)}...`
+            : raw;
         rawUpdateLogger.debug(`telegram update: ${preview}`);
       } catch (err) {
         rawUpdateLogger.debug(`telegram update log failed: ${String(err)}`);
diff --git a/src/agents/cli-runner/session-history.test.ts b/src/agents/cli-runner/session-history.test.ts
@@ -1019,4 +1019,14 @@ describe("buildCliSessionHistoryPrompt", () => {
     expect(prompt).toContain("POST_SUMMARY_TAIL_USER");
     expect(prompt).toContain("POST_SUMMARY_TAIL_ASSISTANT");
   });
+
+  it("does not produce broken surrogates when reseed history contains emoji", async () => {
+    const messages = [
+      makeHistory("user", `Hello from a previous session. ${"a".repeat(200)}🚀`),
+      makeHistory("assistant", "Got it — here is the answer.".repeat(20)),
+    ];
+    const prompt = buildCliSessionHistoryPrompt({ messages, maxHistoryChars: 600 });
+    expect(prompt).toBeDefined();
+    expect(prompt).not.toContain("�");
+  });
 });
diff --git a/src/agents/cli-runner/session-history.ts b/src/agents/cli-runner/session-history.ts
@@ -4,6 +4,7 @@
  */
 import fsp from "node:fs/promises";
 import path from "node:path";
+import { truncateUtf16Safe, sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
 import {
   resolveSessionFilePath,
   resolveSessionFilePathOptions,
@@ -214,8 +215,8 @@ export function buildCliSessionHistoryPrompt(params: {
       0,
       maxHistoryChars - truncationMarker.length - separatorBudget - tailBudget,
     );
-    const summaryTruncated = renderedSummary.slice(0, summaryBudget).trimEnd();
-    const tailTruncated = tailBudget > 0 ? tailRaw.slice(-tailBudget).trimStart() : "";
+    const summaryTruncated = truncateUtf16Safe(renderedSummary, summaryBudget).trimEnd();
+    const tailTruncated = tailBudget > 0 ? sliceUtf16Safe(tailRaw, -tailBudget).trimStart() : "";
     return [truncationMarker, summaryTruncated, tailTruncated].filter(Boolean).join("\n");
   };
 
@@ -242,7 +243,7 @@ export function buildCliSessionHistoryPrompt(params: {
         // reserved tail budget instead of being dropped wholesale.
         renderedHistory = renderTruncatedSummaryWithTail(summaryRendered);
       } else if (tailRaw.length > remainingBudget) {
-        renderedHistory = `${summaryBlock}${truncationMarker}\n${tailRaw.slice(-remainingBudget).trimStart()}`;
+        renderedHistory = `${summaryBlock}${truncationMarker}\n${sliceUtf16Safe(tailRaw, -remainingBudget).trimStart()}`;
       } else {
         renderedHistory = `${summaryBlock}${tailRaw}`;
       }
@@ -253,7 +254,7 @@ export function buildCliSessionHistoryPrompt(params: {
     // (older turns dropped, recent tail retained).
     renderedHistory =
       tailRaw.length > maxHistoryChars
-        ? `${truncationMarker}\n${tailRaw.slice(-maxHistoryChars).trimStart()}`
+        ? `${truncationMarker}\n${sliceUtf16Safe(tailRaw, -maxHistoryChars).trimStart()}`
         : tailRaw;
   }
 
diff --git a/src/tui/tui-local-shell.ts b/src/tui/tui-local-shell.ts
@@ -1,6 +1,7 @@
 // Launches and manages the local shell process used by TUI local mode.
 import { spawn } from "node:child_process";
 import type { Component, OverlayHandle, SelectItem } from "@earendil-works/pi-tui";
+import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
 import { tryProcessCwd } from "../infra/safe-cwd.js";
 import { createSearchableSelectList } from "./components/selectors.js";
 
@@ -114,7 +115,7 @@ export function createLocalShellRunner(deps: LocalShellDeps) {
 
     const appendWithCap = (text: string, chunk: string) => {
       const combined = text + chunk;
-      return combined.length > maxChars ? combined.slice(-maxChars) : combined;
+      return combined.length > maxChars ? sliceUtf16Safe(combined, -maxChars) : combined;
     };
 
     await new Promise<void>((resolve) => {
@@ -143,9 +144,10 @@ export function createLocalShellRunner(deps: LocalShellDeps) {
         // Keep the tail (consistent with the streaming appendWithCap above) so a
         // large stdout cannot evict stderr: the failure reason (FATAL etc.) at the
         // end is what the operator needs most when output overflows the cap.
-        const combined = (stdout + (stderr ? (stdout ? "\n" : "") + stderr : ""))
-          .slice(-maxChars)
-          .trimEnd();
+        const combined = sliceUtf16Safe(
+          stdout + (stderr ? (stdout ? "\n" : "") + stderr : ""),
+          -maxChars,
+        ).trimEnd();
 
         if (combined) {
           for (const lineLocal of combined.split("\n")) {
