Skip to content

Commit 35c4f99

Browse files
steipetesteipete
committed
fix: make compaction and memory truncation UTF-16 safe
1 parent 95b818b commit 35c4f99

6 files changed

Lines changed: 82 additions & 56 deletions

File tree

‎extensions/memory-core/src/short-term-promotion.test.ts‎

Lines changed: 61 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,6 @@ import {
3131
removeGroundedShortTermCandidates,
3232
repairShortTermPromotionArtifacts,
3333
testing,
34-
truncatePromotedSnippet,
35-
truncateShortTermSnippet,
3634
} from "./short-term-promotion.js";
3735

3836
describe("short-term promotion", () => {
@@ -3590,41 +3588,70 @@ describe("short-term promotion", () => {
35903588
});
35913589
});
35923590
});
3593-
});
3594-
3595-
describe("truncateShortTermSnippet", () => {
3596-
it("returns the original snippet when within the char limit", () => {
3597-
expect(truncateShortTermSnippet("short")).toBe("short");
3598-
});
3599-
3600-
it("truncates oversize snippets to the cap", () => {
3601-
const input = "x".repeat(900);
3602-
const result = truncateShortTermSnippet(input);
3603-
expect(result.length).toBeLessThanOrEqual(800);
3604-
});
3591+
describe("UTF-16 snippet bounds", () => {
3592+
it("stores a complete-code-point short-term recall snippet", async () => {
3593+
await withTempWorkspace(async (workspaceDir) => {
3594+
const prefix = "y".repeat(testing.SHORT_TERM_RECALL_MAX_SNIPPET_CHARS - 1);
3595+
await recordShortTermRecalls({
3596+
workspaceDir,
3597+
query: "utf16 recall",
3598+
results: [
3599+
{
3600+
path: "memory/2026-04-03.md",
3601+
source: "memory",
3602+
startLine: 1,
3603+
endLine: 1,
3604+
score: 0.9,
3605+
snippet: `${prefix}🚀tail`,
3606+
},
3607+
],
3608+
});
36053609

3606-
it("does not split a surrogate pair at the cap boundary", () => {
3607-
const input = `${"y".repeat(799)}🚀`;
3608-
const result = truncateShortTermSnippet(input);
3609-
expect(result).not.toContain("�");
3610-
});
3611-
});
3610+
const entries = Object.values(await readRecallStoreEntries(workspaceDir));
3611+
expect(entries).toHaveLength(1);
3612+
expect(readEntrySnippet(entries[0])).toBe(prefix);
3613+
});
3614+
});
36123615

3613-
describe("truncatePromotedSnippet", () => {
3614-
it("returns the original snippet when within token limit", () => {
3615-
const result = truncatePromotedSnippet("short snippet", 100);
3616-
expect(result).toBe("short snippet");
3617-
});
3616+
it("writes a complete-code-point promoted MEMORY.md snippet", async () => {
3617+
await withTempWorkspace(async (workspaceDir) => {
3618+
const prefix = "a".repeat(7);
3619+
const snippet = `${prefix}🚀tail`;
3620+
await writeDailyMemoryNote(workspaceDir, "2026-04-03", [snippet]);
3621+
await recordShortTermRecalls({
3622+
workspaceDir,
3623+
query: "utf16 promotion",
3624+
results: [
3625+
{
3626+
path: "memory/2026-04-03.md",
3627+
source: "memory",
3628+
startLine: 1,
3629+
endLine: 1,
3630+
score: 0.9,
3631+
snippet,
3632+
},
3633+
],
3634+
});
3635+
const ranked = await rankShortTermPromotionCandidates({
3636+
workspaceDir,
3637+
minScore: 0,
3638+
minRecallCount: 0,
3639+
minUniqueQueries: 0,
3640+
});
36183641

3619-
it("truncates and appends ellipsis", () => {
3620-
const input = "x".repeat(500);
3621-
const result = truncatePromotedSnippet(input, 10);
3622-
expect(result.endsWith("...")).toBe(true);
3623-
});
3642+
await applyShortTermPromotions({
3643+
workspaceDir,
3644+
candidates: ranked,
3645+
minScore: 0,
3646+
minRecallCount: 0,
3647+
minUniqueQueries: 0,
3648+
maxPromotedSnippetTokens: 2,
3649+
});
36243650

3625-
it("does not split a surrogate pair at the boundary", () => {
3626-
const input = `aa🚀. ${"b".repeat(300)}`;
3627-
const result = truncatePromotedSnippet(input, 15);
3628-
expect(result).not.toContain("�");
3651+
const memoryText = await fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8");
3652+
expect(memoryText).toContain(`- ${prefix}... [`);
3653+
expect(memoryText).not.toContain("🚀");
3654+
});
3655+
});
36293656
});
36303657
});

‎extensions/memory-core/src/short-term-promotion.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
import { createHash } from "node:crypto";
33
import fs from "node:fs/promises";
44
import path from "node:path";
5-
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
65
import { KeyedAsyncQueue } from "openclaw/plugin-sdk/keyed-async-queue";
76
import type { MemorySearchResult } from "openclaw/plugin-sdk/memory-core-host-runtime-files";
87
import {
@@ -17,6 +16,7 @@ import {
1716
normalizeStringEntries,
1817
uniqueStrings,
1918
} from "openclaw/plugin-sdk/string-coerce-runtime";
19+
import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
2020
import {
2121
deriveConceptTags,
2222
MAX_CONCEPT_TAGS,
@@ -356,7 +356,7 @@ function normalizeSnippet(raw: string): string {
356356
return trimmed.replace(/\s+/g, " ");
357357
}
358358

359-
export function truncateShortTermSnippet(snippet: string): string {
359+
function truncateShortTermSnippet(snippet: string): string {
360360
if (snippet.length <= SHORT_TERM_RECALL_MAX_SNIPPET_CHARS) {
361361
return snippet;
362362
}
@@ -2347,7 +2347,7 @@ function resolvePromotedSnippetCharLimit(maxTokens: number): number {
23472347
return tokenLimit * PROMOTED_SNIPPET_CHARS_PER_TOKEN_ESTIMATE;
23482348
}
23492349

2350-
export function truncatePromotedSnippet(snippet: string, maxTokens: number): string {
2350+
function truncatePromotedSnippet(snippet: string, maxTokens: number): string {
23512351
const limit = resolvePromotedSnippetCharLimit(maxTokens);
23522352
if (limit === 0 || snippet.length <= limit) {
23532353
return snippet;
@@ -2365,7 +2365,7 @@ export function truncatePromotedSnippet(snippet: string, maxTokens: number): str
23652365
: wordBoundary >= Math.floor(limit * 0.65)
23662366
? wordBoundary
23672367
: limit;
2368-
return `${truncateUtf16Safe(hardLimit, cutAt).trimEnd()}...`;
2368+
return `${hardLimit.slice(0, cutAt).trimEnd()}...`;
23692369
}
23702370

23712371
function formatPromotedSnippetForMemory(rawSnippet: string, maxTokens: number): string {

‎packages/agent-core/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@
9696
},
9797
"dependencies": {
9898
"@openclaw/ai": "workspace:*",
99+
"@openclaw/normalization-core": "workspace:*",
99100
"typebox": "1.3.3"
100101
}
101102
}

‎packages/agent-core/src/harness/compaction/utils.test.ts‎

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { describe, expect, it } from "vitest";
22
import type { Message } from "../../../../llm-core/src/index.js";
3-
import { serializeConversation, truncateForSummary } from "./utils.js";
3+
import { serializeConversation } from "./utils.js";
44

55
describe("serializeConversation", () => {
66
it.each([
@@ -33,23 +33,18 @@ describe("serializeConversation", () => {
3333

3434
expect(serializeConversation(messages)).toBe(`[Tool result]: ${expected}`);
3535
});
36-
});
37-
38-
describe("truncateForSummary", () => {
39-
it("returns the original text when it fits within maxChars", () => {
40-
expect(truncateForSummary("hello", 100)).toBe("hello");
41-
});
4236

43-
it("truncates long text with a summary notice", () => {
44-
const input = "x".repeat(200);
45-
const result = truncateForSummary(input, 100);
46-
expect(result.length).toBeLessThan(input.length);
47-
expect(result).toContain("more characters truncated");
48-
});
37+
it("keeps truncated tool results UTF-16 safe and reports the exact omitted count", () => {
38+
const prefix = "a".repeat(1_999);
39+
const messages = [
40+
{
41+
role: "toolResult",
42+
content: [{ type: "toolResult", content: `${prefix}🚀tail` }],
43+
},
44+
] as unknown as Message[];
4945

50-
it("does not split a surrogate pair at the truncation boundary", () => {
51-
const input = `aa🚀${"b".repeat(200)}`;
52-
const result = truncateForSummary(input, 80);
53-
expect(result).not.toContain("�");
46+
expect(serializeConversation(messages)).toBe(
47+
`[Tool result]: ${prefix}\n\n[... 6 more characters truncated]`,
48+
);
5449
});
5550
});

‎packages/agent-core/src/harness/compaction/utils.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ function safeJsonStringify(value: unknown): string {
102102
}
103103
}
104104

105-
export function truncateForSummary(text: string, maxChars: number): string {
105+
function truncateForSummary(text: string, maxChars: number): string {
106106
if (text.length <= maxChars) {
107107
return text;
108108
}

‎pnpm-lock.yaml‎

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)