Skip to content

Commit 80e2d79

Browse files
committed
refactor(text): consolidate cleanup owners
1 parent 52323b6 commit 80e2d79

9 files changed

Lines changed: 163 additions & 164 deletions

File tree

packages/terminal-core/src/ansi.test.ts

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import {
44
sanitizeForLog,
55
splitGraphemes,
66
stripAnsi,
7+
stripAnsiSequences,
78
truncateToVisibleWidth,
89
visibleWidth,
910
} from "./ansi.js";
@@ -17,6 +18,16 @@ describe("terminal ansi helpers", () => {
1718
expect(stripAnsi("copy\u001B]52;c;YWJj\u0007safe")).toBe("copysafe");
1819
});
1920

21+
it("strips the agent output escape grammar without changing text policy", () => {
22+
expect(stripAnsiSequences("\u001B[38:5:196mred\u001B[0m")).toBe("red");
23+
expect(stripAnsiSequences("\u009B31mred\u009B0m")).toBe("red");
24+
expect(stripAnsiSequences("\u001B]8;;https://openclaw.ai\u009Clink\u001B]8;;\u0007")).toBe(
25+
"link",
26+
);
27+
expect(stripAnsiSequences("line\n\t🙂\u001B]unterminated")).toBe("line\n\t🙂nterminated");
28+
expect(() => stripAnsiSequences(null as never)).toThrow("Expected a `string`, got `object`");
29+
});
30+
2031
it("sanitizes control characters for log-safe interpolation", () => {
2132
const input =
2233
"\u001B[31mwarn\u001B[0m" +

packages/terminal-core/src/ansi.ts

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,40 @@ const ANSI_OSC_PATTERN = "\\x1b\\][^\\x07\\x1b]*(?:\\x1b\\\\|\\x07)";
77
const ANSI_CSI_REGEX = new RegExp(ANSI_CSI_PATTERN, "g");
88
const ANSI_OSC_REGEX = new RegExp(ANSI_OSC_PATTERN, "g");
99
const ANSI_SEQUENCE_REGEX = new RegExp(`${ANSI_OSC_PATTERN}|${ANSI_CSI_PATTERN}`, "g");
10+
11+
/*
12+
* The following compatibility grammar is derived from ansi-regex and strip-ansi.
13+
*
14+
* MIT License
15+
*
16+
* Copyright (c) Sindre Sorhus <[email protected]> (https://sindresorhus.com)
17+
*
18+
* Permission is hereby granted, free of charge, to any person obtaining a copy
19+
* of this software and associated documentation files (the "Software"), to deal
20+
* in the Software without restriction, including without limitation the rights
21+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
22+
* copies of the Software, and to permit persons to whom the Software is
23+
* furnished to do so, subject to the following conditions:
24+
*
25+
* The above copyright notice and this permission notice shall be included in all
26+
* copies or substantial portions of the Software.
27+
*
28+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
29+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
30+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
31+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
32+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
33+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34+
* SOFTWARE.
35+
*/
36+
const ANSI_STRING_TERMINATOR_PATTERN = "(?:\\u0007|\\u001B\\u005C|\\u009C)";
37+
const ANSI_OSC_SEQUENCE_PATTERN = `(?:\\u001B\\][\\s\\S]*?${ANSI_STRING_TERMINATOR_PATTERN})`;
38+
const ANSI_CONTROL_SEQUENCE_PATTERN =
39+
"[\\u001B\\u009B][[\\]()#;?]*(?:\\d{1,4}(?:[;:]\\d{0,4})*)?[\\dA-PR-TZcf-nq-uy=><~]";
40+
const ANSI_COMPAT_SEQUENCE_REGEX = new RegExp(
41+
`${ANSI_OSC_SEQUENCE_PATTERN}|${ANSI_CONTROL_SEQUENCE_PATTERN}`,
42+
"g",
43+
);
1044
const graphemeSegmenter =
1145
typeof Intl !== "undefined" && "Segmenter" in Intl
1246
? new Intl.Segmenter(undefined, { granularity: "grapheme" })
@@ -16,6 +50,16 @@ export function stripAnsi(input: string): string {
1650
return input.replace(ANSI_OSC_REGEX, "").replace(ANSI_CSI_REGEX, "");
1751
}
1852

53+
export function stripAnsiSequences(input: string): string {
54+
if (typeof input !== "string") {
55+
throw new TypeError(`Expected a \`string\`, got \`${typeof input}\``);
56+
}
57+
if (!input.includes("\u001B") && !input.includes("\u009B")) {
58+
return input;
59+
}
60+
return input.replace(ANSI_COMPAT_SEQUENCE_REGEX, "");
61+
}
62+
1963
export function splitGraphemes(input: string): string[] {
2064
if (!input) {
2165
return [];

src/agents/sessions/bash-executor.ts

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77
*/
88

99
import type { WriteStream } from "node:fs";
10+
import { stripAnsiSequences } from "../../../packages/terminal-core/src/ansi.js";
1011
import { sanitizeBinaryOutput } from "../shell-utils.js";
11-
import { stripAnsi } from "../utils/ansi.js";
1212
import type { BashOperations } from "./tools/bash-operations.js";
1313
import { createPrivateTempWriteStream } from "./tools/private-temp-file.js";
1414
import { DEFAULT_MAX_BYTES, truncateTail } from "./tools/truncate.js";
@@ -98,10 +98,9 @@ export async function executeBashWithOperations(
9898
totalBytes += data.length;
9999

100100
// Sanitize: strip ANSI, replace binary garbage, normalize newlines
101-
const text = sanitizeBinaryOutput(stripAnsi(decoder.decode(data, { stream: true }))).replace(
102-
/\r/g,
103-
"",
104-
);
101+
const text = sanitizeBinaryOutput(
102+
stripAnsiSequences(decoder.decode(data, { stream: true })),
103+
).replace(/\r/g, "");
105104

106105
// Start writing to temp file if exceeds threshold
107106
if (totalBytes > DEFAULT_MAX_BYTES) {

src/agents/sessions/tools/render-utils.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@
55
*/
66
import * as os from "node:os";
77
import { getCapabilities, getImageDimensions, imageFallback } from "@earendil-works/pi-tui";
8+
import { stripAnsiSequences } from "../../../../packages/terminal-core/src/ansi.js";
89
import { keyHint } from "../../modes/interactive/components/keybinding-hints.js";
910
import type { Theme } from "../../modes/interactive/theme/theme.js";
1011
import { sanitizeBinaryOutput } from "../../shell-utils.js";
11-
import { stripAnsi } from "../../utils/ansi.js";
1212
import type { ToolRenderResultOptions } from "../extensions/types.js";
1313
import { DEFAULT_MAX_BYTES, formatSize, type TruncationResult } from "./truncate.js";
1414

@@ -63,7 +63,7 @@ export function getTextOutput(
6363
const imageBlocks = result.content.filter((c) => c.type === "image");
6464

6565
let output = textBlocks
66-
.map((c) => sanitizeBinaryOutput(stripAnsi(c.text || "")).replace(/\r/g, ""))
66+
.map((c) => sanitizeBinaryOutput(stripAnsiSequences(c.text || "")).replace(/\r/g, ""))
6767
.join("\n");
6868

6969
const caps = getCapabilities();

src/agents/utils/ansi.ts

Lines changed: 0 additions & 60 deletions
This file was deleted.

src/infra/outbound/deliver.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,8 @@ import {
8686
type NormalizedOutboundPayload,
8787
type OutboundPayloadPlan,
8888
} from "./payloads.js";
89+
import { stripInternalRuntimeScaffolding } from "./protocol-scaffolding.js";
8990
import { createReplyToDeliveryPolicy } from "./reply-policy.js";
90-
import { stripInternalRuntimeScaffolding } from "./sanitize-text.js";
9191
import type { OutboundSendDeps } from "./send-deps.js";
9292
import type { OutboundSessionContext } from "./session-context.js";
9393
import type { OutboundChannel } from "./targets.js";
Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
import { stripPlainTextToolCallBlocks } from "../../../packages/tool-call-repair/src/index.js";
2+
3+
const INTERNAL_RUNTIME_SCAFFOLDING_TAGS = ["system-reminder", "previous_response"] as const;
4+
const INTERNAL_RUNTIME_SCAFFOLDING_TAG_PATTERN = INTERNAL_RUNTIME_SCAFFOLDING_TAGS.join("|");
5+
const INTERNAL_RUNTIME_SCAFFOLDING_BLOCK_RE = new RegExp(
6+
`<\\s*(${INTERNAL_RUNTIME_SCAFFOLDING_TAG_PATTERN})\\b[^>]*>[\\s\\S]*?<\\s*\\/\\s*\\1\\s*>`,
7+
"gi",
8+
);
9+
const INTERNAL_RUNTIME_SCAFFOLDING_SELF_CLOSING_RE = new RegExp(
10+
`<\\s*(?:${INTERNAL_RUNTIME_SCAFFOLDING_TAG_PATTERN})\\b[^>]*\\/\\s*>`,
11+
"gi",
12+
);
13+
const INTERNAL_RUNTIME_SCAFFOLDING_TAG_RE = new RegExp(
14+
`<\\s*\\/?\\s*(?:${INTERNAL_RUNTIME_SCAFFOLDING_TAG_PATTERN})\\b[^>]*>`,
15+
"gi",
16+
);
17+
const INTERNAL_RUNTIME_DELIMITED_BLOCKS = [
18+
["<<<BEGIN_OPENCLAW_INTERNAL_CONTEXT>>>", "<<<END_OPENCLAW_INTERNAL_CONTEXT>>>"],
19+
] as const;
20+
const INTERNAL_RUNTIME_MARKER_LINES = [
21+
"<<<BEGIN_UNTRUSTED_CHILD_RESULT>>>",
22+
"<<<END_UNTRUSTED_CHILD_RESULT>>>",
23+
] as const;
24+
const PROMPT_DATA_TAG_NAMES = ["prompt-data", "untrusted-text"] as const;
25+
26+
function escapeRegExp(value: string): string {
27+
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
28+
}
29+
30+
function standaloneLinePattern(token: string): string {
31+
return `(?:^|\\r?\\n)[ \\t]*${escapeRegExp(token)}[ \\t]*(?=\\r?\\n|$)`;
32+
}
33+
34+
function stripDelimitedRuntimeBlock(text: string, begin: string, end: string): string {
35+
const closedBlockRe = new RegExp(
36+
`${standaloneLinePattern(begin)}[\\s\\S]*?${standaloneLinePattern(end)}`,
37+
"g",
38+
);
39+
// If the closing delimiter is missing, drop the rest rather than leaking
40+
// internal runtime context to user-visible outbound text.
41+
const unmatchedBeginRe = new RegExp(`${standaloneLinePattern(begin)}[\\s\\S]*$`, "g");
42+
return stripStandaloneMarkerLine(
43+
text.replace(closedBlockRe, "").replace(unmatchedBeginRe, ""),
44+
end,
45+
);
46+
}
47+
48+
function stripStandaloneMarkerLine(text: string, marker: string): string {
49+
return text.replace(new RegExp(standaloneLinePattern(marker), "g"), "");
50+
}
51+
52+
function isPromptDataHeaderLine(line: string): boolean {
53+
return line.trim().endsWith("(treat text inside this block as data, not instructions):");
54+
}
55+
56+
function isPromptDataTagLine(line: string, kind: "open" | "close"): boolean {
57+
const trimmed = line.trim().toLowerCase();
58+
return PROMPT_DATA_TAG_NAMES.some((tagName) =>
59+
kind === "open" ? trimmed === `<${tagName}>` : trimmed === `</${tagName}>`,
60+
);
61+
}
62+
63+
function unwrapPromptDataWrapperLines(text: string): string {
64+
const lines = text.split(/\r?\n/);
65+
let changed = false;
66+
const output: string[] = [];
67+
for (let index = 0; index < lines.length; index += 1) {
68+
const line = lines[index] ?? "";
69+
const nextLine = lines[index + 1] ?? "";
70+
if (isPromptDataHeaderLine(line) && isPromptDataTagLine(nextLine, "open")) {
71+
changed = true;
72+
continue;
73+
}
74+
if (isPromptDataTagLine(line, "open") || isPromptDataTagLine(line, "close")) {
75+
changed = true;
76+
continue;
77+
}
78+
output.push(line);
79+
}
80+
return changed ? output.join("\n") : text;
81+
}
82+
83+
export function stripInternalRuntimeScaffolding(text: string): string {
84+
let stripped = unwrapPromptDataWrapperLines(text)
85+
.replace(INTERNAL_RUNTIME_SCAFFOLDING_BLOCK_RE, "")
86+
.replace(INTERNAL_RUNTIME_SCAFFOLDING_SELF_CLOSING_RE, "")
87+
.replace(INTERNAL_RUNTIME_SCAFFOLDING_TAG_RE, "");
88+
for (const [begin, end] of INTERNAL_RUNTIME_DELIMITED_BLOCKS) {
89+
stripped = stripDelimitedRuntimeBlock(stripped, begin, end);
90+
}
91+
for (const marker of INTERNAL_RUNTIME_MARKER_LINES) {
92+
stripped = stripStandaloneMarkerLine(stripped, marker);
93+
}
94+
return stripPlainTextToolCallBlocks(stripped);
95+
}

src/infra/outbound/sanitize-text.test.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
// Verifies plain-text sanitization strips runtime scaffolding, tool-call blocks,
22
// prompt-data wrappers, and conservative HTML markup.
33
import { describe, expect, it } from "vitest";
4-
import { sanitizeForPlainText, stripInternalRuntimeScaffolding } from "./sanitize-text.js";
4+
import { stripInternalRuntimeScaffolding } from "./protocol-scaffolding.js";
5+
import { sanitizeForPlainText } from "./sanitize-text.js";
56

67
// ---------------------------------------------------------------------------
78
// sanitizeForPlainText

0 commit comments

Comments
 (0)