Skip to content

Commit a9582a1

Browse files
steipetewm0018lsr911maweibin
authored
fix: keep bounded text truncation UTF-16 safe (#101654)
* fix: keep bounded text UTF-16 safe Co-authored-by: wm0018 <[email protected]> Co-authored-by: 廖石荣 0668000909 <[email protected]> Co-authored-by: 0668000787 <[email protected]> * chore: keep UTF-16 release note in PR body --------- Co-authored-by: wm0018 <[email protected]> Co-authored-by: 廖石荣 0668000909 <[email protected]> Co-authored-by: 0668000787 <[email protected]>
1 parent 60f0749 commit a9582a1

44 files changed

Lines changed: 347 additions & 42 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎extensions/active-memory/index.test.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,16 @@ vi.mock("openclaw/plugin-sdk/session-store-runtime", async () => {
6161
});
6262

6363
describe("active-memory plugin", () => {
64+
it("keeps previous-message query context UTF-16 well-formed", () => {
65+
const query = testing.buildSearchQuery({
66+
latestUserMessage: "why?",
67+
recentTurns: [{ role: "user", text: `${"x".repeat(119)}🚀tail` }],
68+
});
69+
70+
expect(query).toBe(`${"x".repeat(119)} why?`);
71+
expect(query).not.toContain("\uD83D");
72+
});
73+
6474
const hooks: Record<string, Function> = {};
6575
const hookOptions: Record<string, Record<string, unknown> | undefined> = {};
6676
const registeredCommands: Record<string, any> = {};

‎extensions/active-memory/index.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2735,11 +2735,10 @@ function buildSearchQuery(params: {
27352735
if (!previousUser) {
27362736
return latest || clampSearchQuery(params.latestUserMessage);
27372737
}
2738-
const context = clampSearchQuery(
2739-
normalizeSearchQueryText(stripRecalledContextNoise(previousUser.text)),
2740-
)
2741-
.slice(0, 120)
2742-
.trim();
2738+
const context = truncateUtf16Safe(
2739+
clampSearchQuery(normalizeSearchQueryText(stripRecalledContextNoise(previousUser.text))),
2740+
120,
2741+
).trim();
27432742
return clampSearchQuery(context ? `${context} ${latest}` : latest);
27442743
}
27452744

@@ -3755,6 +3754,7 @@ export default definePluginEntry({
37553754
});
37563755

37573756
const testing = {
3757+
buildSearchQuery,
37583758
buildCacheKey,
37593759
buildCircuitBreakerKey,
37603760
buildMetadata,

‎extensions/memory-lancedb/index.test.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2955,7 +2955,7 @@ describe("memory plugin e2e", () => {
29552955
const fakeRows = [
29562956
{
29572957
id: fakeUuid1,
2958-
text: "User prefers dark mode",
2958+
text: `${"x".repeat(59)}🚀tail`,
29592959
category: "preference",
29602960
vector: [0.1],
29612961
importance: 0.8,
@@ -3035,6 +3035,8 @@ describe("memory plugin e2e", () => {
30353035
const text = result.content?.[0]?.text ?? "";
30363036
expect(text).toContain(fakeUuid1);
30373037
expect(text).toContain(fakeUuid2);
3038+
expect(text).toContain(`- [${fakeUuid1}] ${"x".repeat(59)}...`);
3039+
expect(text).not.toContain("\uD83D");
30383040
// Ensure truncated 8-char prefix alone is NOT the format used
30393041
expect(text).not.toMatch(/\[890e1fae\]/);
30403042
expect(text).not.toMatch(/\[a1b2c3d4\]/);

‎extensions/memory-lancedb/index.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1658,7 +1658,7 @@ export default definePluginEntry({
16581658
});
16591659

16601660
return {
1661-
content: [{ type: "text", text: `Stored: "${text.slice(0, 100)}..."` }],
1661+
content: [{ type: "text", text: `Stored: "${truncateUtf16Safe(text, 100)}..."` }],
16621662
details: { action: "created", id: entry.id },
16631663
};
16641664
},
@@ -1709,7 +1709,7 @@ export default definePluginEntry({
17091709
}
17101710

17111711
const list = results
1712-
.map((r) => `- [${r.entry.id}] ${r.entry.text.slice(0, 60)}...`)
1712+
.map((r) => `- [${r.entry.id}] ${truncateUtf16Safe(r.entry.text, 60)}...`)
17131713
.join("\n");
17141714

17151715
// Strip vector data for serialization

‎extensions/zalouser/src/zalo-js.credentials.test.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,16 @@ import {
3232
sendZaloLink,
3333
sendZaloReaction,
3434
startZaloQrLogin,
35+
testing,
3536
waitForZaloQrLogin,
3637
} from "./zalo-js.js";
3738

39+
describe("Zalo payload bounds", () => {
40+
it("keeps the 2,000-code-unit transport payload UTF-16 well-formed", () => {
41+
expect(testing.truncatePayloadText(`${"x".repeat(1_999)}🚀tail`)).toBe("x".repeat(1_999));
42+
});
43+
});
44+
3845
type StoredCredentialFile = {
3946
imei: string;
4047
cookie: Credentials["cookie"];

‎extensions/zalouser/src/zalo-js.ts‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import {
2626
normalizeOptionalLowercaseString,
2727
normalizeOptionalString,
2828
} from "openclaw/plugin-sdk/string-coerce-runtime";
29-
import { sleep } from "openclaw/plugin-sdk/text-utility-runtime";
29+
import { sleep, truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
3030
import { normalizeZaloReactionIcon } from "./reaction.js";
3131
import { createZalouserSendReceipt } from "./send-receipt.js";
3232
import type {
@@ -986,7 +986,12 @@ function toInboundMessage(message: Message, ownUserId?: string): ZaloInboundMess
986986
};
987987
}
988988

989+
function truncatePayloadText(text: string): string {
990+
return truncateUtf16Safe(text, 2000);
991+
}
992+
989993
export const testing = {
994+
truncatePayloadText,
990995
toInboundMessage,
991996
readCachedGroupContext,
992997
writeCachedGroupContext,
@@ -1238,7 +1243,7 @@ export async function sendZaloTextMessage(
12381243
contentType: media.contentType,
12391244
kind: media.kind,
12401245
});
1241-
const payloadText = (text || options.caption || "").slice(0, 2000);
1246+
const payloadText = truncatePayloadText(text || options.caption || "");
12421247
const textStyles = clampTextStyles(payloadText, options.textStyles);
12431248

12441249
if (media.kind === "audio") {
@@ -1313,7 +1318,7 @@ export async function sendZaloTextMessage(
13131318
};
13141319
}
13151320

1316-
const payloadText = text.slice(0, 2000);
1321+
const payloadText = truncatePayloadText(text);
13171322
const textStyles = clampTextStyles(payloadText, options.textStyles);
13181323
const response = await api.sendMessage(
13191324
textStyles ? { msg: payloadText, styles: textStyles } : payloadText,

‎src/agents/command/attempt-execution.error-propagation.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,18 @@ describe("ACP diagnostic events", () => {
123123
expect(String(event?.data.text)).not.toContain("sk-abcdefghijklmnopqrstuvwxyz123456");
124124
});
125125

126+
it("keeps bounded ACP diagnostics UTF-16 well-formed", () => {
127+
emitAcpRuntimeEvent({
128+
runId: "run-utf16-status",
129+
event: {
130+
type: "status",
131+
text: `${"x".repeat(239)}🚀tail`,
132+
},
133+
});
134+
135+
expect(captured[0]?.data.text).toBe("x".repeat(239));
136+
});
137+
126138
it("keeps audit-only ACP lifecycle and runtime events off the shared bus", () => {
127139
const secret = "private ACP cause chain";
128140
const auditEvents: AgentEventPayload[] = [];

‎src/agents/command/attempt-execution.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import {
66
normalizeOptionalLowercaseString,
77
type FastMode,
88
} from "@openclaw/normalization-core/string-coerce";
9+
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
910
import { sanitizeForLog } from "../../../packages/terminal-core/src/ansi.js";
1011
import { ACP_TURN_TIMEOUT_DETAIL_CODE } from "../../acp/control-plane/manager.turn-timeout.js";
1112
import { formatAcpErrorChain } from "../../acp/runtime/errors.js";
@@ -1162,7 +1163,7 @@ function resolvePresentProxyEnvKeys(env: NodeJS.ProcessEnv = process.env): strin
11621163
}
11631164

11641165
function sanitizeAcpDiagnosticText(value: string): string {
1165-
return redactSensitiveText(value).replace(/\s+/g, " ").trim().slice(0, 240);
1166+
return truncateUtf16Safe(redactSensitiveText(value).replace(/\s+/g, " ").trim(), 240);
11661167
}
11671168

11681169
function acpRuntimeEventDiagnostics(event: AcpRuntimeEvent): Record<string, unknown> {

‎src/agents/openai-transport-stream.test.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,15 @@ function expectRecordFields(record: unknown, expected: Record<string, unknown>)
142142
}
143143

144144
describe("openai transport stream", () => {
145+
it("keeps bounded redacted diagnostics UTF-16 well-formed", () => {
146+
const payload = testing.stringifyRedactedPayload(`${"x".repeat(7_998)}🚀tail`);
147+
const event = testing.stringifyRedactedEvent(`${"x".repeat(1_998)}🚀tail`);
148+
149+
expect(payload).toContain(`${"x".repeat(7_998)}…<truncated>`);
150+
expect(event).toContain(`${"x".repeat(1_998)}…<truncated>`);
151+
expect(payload).not.toContain("\uD83D");
152+
expect(event).not.toContain("\uD83D");
153+
});
145154
it("fails Azure Responses streams when headers arrive but no first event follows", async () => {
146155
vi.useFakeTimers();
147156
try {

‎src/agents/openai-transport-stream.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ import {
4848
import { isRecord } from "@openclaw/normalization-core/record-coerce";
4949
import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce";
5050
import { uniqueStrings } from "@openclaw/normalization-core/string-normalization";
51+
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
5152
import OpenAI, { AzureOpenAI } from "openai";
5253
import type { ChatCompletionChunk } from "openai/resources/chat/completions.js";
5354
import type {
@@ -462,15 +463,15 @@ function stringifyRedactedPayload(value: unknown): string {
462463
return "<empty>";
463464
}
464465
const redacted = redactSensitiveText(encoded, { mode: "tools" });
465-
return redacted.length > 8000 ? `${redacted.slice(0, 8000)}…<truncated>` : redacted;
466+
return redacted.length > 8000 ? `${truncateUtf16Safe(redacted, 8000)}…<truncated>` : redacted;
466467
} catch {
467468
return "<unserializable>";
468469
}
469470
}
470471

471472
function stringifyRedactedEvent(value: unknown): string {
472473
const redacted = stringifyRedactedPayload(value);
473-
return redacted.length > 2000 ? `${redacted.slice(0, 2000)}…<truncated>` : redacted;
474+
return redacted.length > 2000 ? `${truncateUtf16Safe(redacted, 2000)}…<truncated>` : redacted;
474475
}
475476

476477
type ResponsesFailedNoDetailsObservation = {
@@ -4529,5 +4530,7 @@ export const testing = {
45294530
summarizeResponsesFailedNoDetailsObservation,
45304531
summarizeResponsesPayload,
45314532
summarizeResponsesTools,
4533+
stringifyRedactedEvent,
4534+
stringifyRedactedPayload,
45324535
};
45334536
export { testing as __testing };

0 commit comments

Comments
 (0)