Skip to content

Commit 2c99e1a

Browse files
committed
fix(telemetry): bound shared ANSI normalization
Use a shared ANSI entry point with explicit template and legacy profiles. Bound raw first-line processing while preserving historical maskString output and error fingerprints. Add input-limit, malformed ANSI and fingerprint compatibility regressions. Refs #1588
1 parent f4cf548 commit 2c99e1a

5 files changed

Lines changed: 201 additions & 10 deletions

File tree

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
import stripAnsi from "strip-ansi"
2+
3+
// JavaScript string lengths are UTF-16 code units, matching maskString's historical limits.
4+
export const MAX_INPUT = 8192
5+
export const LOOKAHEAD = 2048
6+
7+
/**
8+
* Shared bounded ANSI entry point with explicit compatibility profiles.
9+
* `legacy` preserves maskString's 7-bit rules and existing lookahead window, so hashes
10+
* do not change. `template` handles the broader strip-ansi grammar on the original first
11+
* line. An oversized first line returns undefined: callers must redact the whole line,
12+
* rather than emit a prefix whose ANSI sequence or credential marker may have been cut.
13+
*/
14+
export function normalize(raw: string, profile: "legacy"): string
15+
export function normalize(raw: string, profile: "template"): string | undefined
16+
export function normalize(raw: string, profile: "legacy" | "template"): string | undefined {
17+
if (profile === "legacy") {
18+
// maskString already supplies at most MAX_INPUT + LOOKAHEAD code units. Keep this
19+
// historical regex frozen; adopting the template profile here would change hashes.
20+
return raw
21+
.slice(0, MAX_INPUT + LOOKAHEAD)
22+
.replace(/\x1b(?:\[[0-?]*[ -\/]*[@-~]|\][^\x07\x1b]*(?:\x07|\x1b\\))/g, "")
23+
}
24+
// One extra code unit distinguishes an exact-limit line followed by a newline from
25+
// an oversized line. Never scan the unbounded input or strip across a line boundary.
26+
const first = raw.slice(0, MAX_INPUT + 1).split(/[\r\n\u2028\u2029]/, 1)[0]
27+
if (first.length > MAX_INPUT) return undefined
28+
return stripAnsi(first)
29+
}
30+
31+
export * as Ansi from "./ansi"

‎packages/opencode/src/altimate/telemetry/error-detail.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { Telemetry } from "./index"
2-
import stripAnsi from "strip-ansi"
2+
import { Ansi } from "./ansi"
33

44
const MAX_MESSAGE = 200
55
const PLACEHOLDER = "<x>"
@@ -414,12 +414,14 @@ export const ERROR_VOCABULARY: ReadonlySet<string> = new Set([
414414
* rest of the line become one `<x>` too, because a credential value may itself consist
415415
* of vocabulary words. This deliberately loses diagnostic text after ambiguous words
416416
* such as key or session: privacy takes priority over template detail. The output holds
417-
* only vocabulary words, `<x>` and spaces.
417+
* only vocabulary words, `<x>` and spaces. Raw first lines over 8192 UTF-16 code units
418+
* become `<x>` in full; partially normalized prefixes could cut ANSI or credential keys.
418419
*/
419420
export function templateErrorMessage(raw: string): string {
420-
// Split first so an OSC sequence spanning a newline cannot erase the line boundary.
421+
const line = Ansi.normalize(raw, "template")
422+
if (line === undefined) return PLACEHOLDER
421423
// Use the tokenizer's letter boundaries; spaced keys require an assignment separator.
422-
const first = stripAnsi(raw.split(/[\r\n\u2028\u2029]/, 1)[0])
424+
const first = line
423425
.replace(/(?<![A-Za-z])connection[_-]string(?![A-Za-z])/gi, "connectionstring")
424426
.replace(/(?<![A-Za-z])connection[ \t]+string["'`]?[ \t]*(?=[:=])/gi, "connectionstring")
425427
const out: string[] = []

‎packages/opencode/src/altimate/telemetry/index.ts‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { getOrCreateMachineId } from "@/altimate/util/machine-id"
88
import { TelemetryIdentity } from "./identity"
99
import { OptOutMarker } from "./opt-out"
1010
import { TelemetryConfig } from "./config-state"
11+
import { Ansi } from "./ansi"
1112
import { createHash, randomUUID } from "crypto"
1213
// altimate_change — first-run health: the event-loop stall monitor labels which thread stalled
1314
import { isMainThread } from "node:worker_threads"
@@ -161,8 +162,8 @@ const pmTail = (sep: string, term: string) => pmSpan(sep) + "*" + pmChunks(sep)
161162
const PM_NOT_ROUTE = "(?<!(?:GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS|route|endpoint)\\s)"
162163
// input cap and the lookahead the rules may see past it (wider than any
163164
// proof scan — all are bounded at the 255-byte component limit)
164-
const PM_CAP = 8192
165-
const PM_LOOKAHEAD = 2048
165+
const PM_CAP = Ansi.MAX_INPUT
166+
const PM_LOOKAHEAD = Ansi.LOOKAHEAD
166167
// is a `q`-quoted span still open at the end of `s`? Mirrors the quote rules'
167168
// grammar exactly (`q(?:[^q\\]|\\.)*q`): a backslash escapes only inside a span
168169
function pmQuoteOpen(s: string, q: string): boolean {
@@ -1710,10 +1711,8 @@ export namespace Telemetry {
17101711

17111712
// the masking chain proper, on bounded input (see maskString)
17121713
function pmMask(s: string, opts?: { maskPaths?: boolean }): string {
1713-
let out = s
1714-
// ANSI CSI sequences (colored subprocess stderr) would otherwise split
1715-
// tokens so neither credential nor path rules can see them
1716-
.replace(/\x1b(?:\[[0-?]*[ -\/]*[@-~]|\][^\x07\x1b]*(?:\x07|\x1b\\))/g, "")
1714+
// The shared helper's legacy profile preserves historical maskString/hash behavior.
1715+
let out = Ansi.normalize(s, "legacy")
17171716
.replace(/sk-(?:ant-)?[A-Za-z0-9_-]{20,}/g, "sk-***")
17181717
.replace(/Bearer\s+[A-Za-z0-9._-]{20,}/gi, "Bearer ***")
17191718
.replace(/'(?:[^'\\]|\\.)*'/g, "?")
Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
import { describe, expect, test } from "bun:test"
2+
import { createHash } from "node:crypto"
3+
import { Telemetry } from "../../src/altimate/telemetry"
4+
import { ErrorDetail } from "../../src/altimate/telemetry/error-detail"
5+
import { Ansi } from "../../src/altimate/telemetry/ansi"
6+
7+
const cases = [
8+
["plain", "request failed: password is connection refused"],
9+
["csi", "request failed: \x1b[31mpassword\x1b[0m is connection refused"],
10+
["c1", "request failed: \x9b31mpassword\x9b0m is connection refused"],
11+
["osc-bel", "\x1b]0;synthetic title\x07password=wrong user"],
12+
["osc-st", "\x1b]0;synthetic title\x1b\\password=wrong user"],
13+
["osc-c1-st", "\x1b]0;synthetic title\x9cpassword=wrong user"],
14+
["incomplete-osc", "request failed \x1b]8;;wrong user"],
15+
["incomplete-csi", "password=wrong user \x1b[31"],
16+
["multiline", "connection refused\npassword=wrong user"],
17+
["at-cap", "error ".repeat(1638) + "xx"],
18+
["beyond-cap", "error ".repeat(2000) + "password=wrong user"],
19+
["open-quote", "error ".repeat(1300) + '"' + "connection ".repeat(500)],
20+
["path-cut", "error ".repeat(1300) + "/Users/synthetic/a ".repeat(200)],
21+
["lookahead", "x ".repeat(4093) + "password=wrong user"],
22+
["unbroken", "x".repeat(12000)],
23+
]
24+
25+
const pieces = [
26+
"connection refused",
27+
"password=wrong user",
28+
"'wrong user'",
29+
'"connection refused"',
30+
"/Users/synthetic/file.sql",
31+
"C:\\Users\\synthetic\\file.sql",
32+
"\x1b[31m",
33+
"\x9b31m",
34+
"\x1b]8;;https://example.test\x07",
35+
"\n",
36+
"token=wrong user",
37+
"connection string=wrong user",
38+
]
39+
const corpus = Array.from({ length: 96 }, (_, i) =>
40+
pieces
41+
.map((_, j) => pieces[(i * (j + 1) + j * 7) % pieces.length])
42+
.join(i % 2 ? " " : "_")
43+
.repeat(i % 5 === 0 ? 300 : 1),
44+
)
45+
46+
const hashes: Record<string, string> = {
47+
plain: "ae7e754cfb2d72a2",
48+
csi: "ae7e754cfb2d72a2",
49+
c1: "40678db384156848",
50+
"osc-bel": "914eb30c45c8b094",
51+
"osc-st": "914eb30c45c8b094",
52+
"osc-c1-st": "bd974ba379a9b36f",
53+
"incomplete-osc": "821db0d35dce78e7",
54+
"incomplete-csi": "bb977ba58d3d2d21",
55+
multiline: "7ee1b11bcf0c4db1",
56+
"at-cap": "80b18d6fbcfa3063",
57+
"beyond-cap": "80b18d6fbcfa3063",
58+
"open-quote": "80b18d6fbcfa3063",
59+
"path-cut": "80b18d6fbcfa3063",
60+
lookahead: "bf3ce936232f73c1",
61+
unbroken: "e3b0c44298fc1c14",
62+
}
63+
64+
describe("bounded ANSI profiles", () => {
65+
test("shares the historical input and lookahead limits", () => {
66+
expect(Ansi.MAX_INPUT).toBe(8192)
67+
expect(Ansi.LOOKAHEAD).toBe(2048)
68+
})
69+
70+
test.each([
71+
["CSI", "\x1b[31mpassword\x1b[0m=wrong user"],
72+
["C1 CSI", "\x9b31mpassword\x9b0m=wrong user"],
73+
["OSC BEL", "\x1b]0;synthetic\x07password=wrong user"],
74+
["OSC ST", "\x1b]0;synthetic\x1b\\password=wrong user"],
75+
["OSC C1 ST", "\x1b]0;synthetic\x9cpassword=wrong user"],
76+
])("normalizes the template's %s sequences", (_, raw) => {
77+
expect(Ansi.normalize(raw, "template")).toBe("password=wrong user")
78+
})
79+
80+
test.each([
81+
["text", "x".repeat(8193)],
82+
["ANSI-only text", "\x1b[0m".repeat(3000)],
83+
["unfinished CSI", "\x1b[" + "0;".repeat(5000)],
84+
["OSC closing past the cap", "\x1b]0;" + "x".repeat(8192) + "\x07"],
85+
["credential marker crossing the cap", " ".repeat(8190) + "password=wrong user"],
86+
["large unfinished OSC", "\x1b]0;" + "x".repeat(2_000_000)],
87+
])("rejects oversized %s before stripping", (_, raw) => {
88+
expect(Ansi.normalize(raw, "template")).toBeUndefined()
89+
})
90+
91+
test.each(["\n", "\r", "\r\n", "\u2028", "\u2029"])(
92+
"preserves the original first-line boundary for %j",
93+
(newline) => {
94+
const first = "x".repeat(8192)
95+
expect(Ansi.normalize(first + newline + "password=wrong user", "template")).toBe(first)
96+
expect(Ansi.normalize(newline + "x".repeat(100000), "template")).toBe("")
97+
},
98+
)
99+
100+
test("does not emit a surrogate split by the input cap", () => {
101+
expect(Ansi.normalize(" ".repeat(8191) + "😀", "template")).toBeUndefined()
102+
expect(Ansi.normalize(" ".repeat(8190) + "😀", "template")).toBe(" ".repeat(8190) + "😀")
103+
})
104+
105+
test("bounds the legacy normalization window without dropping line separators", () => {
106+
expect(Ansi.normalize("x".repeat(10240) + "wrong user", "legacy")).toBe("x".repeat(10240))
107+
expect(Ansi.normalize("connection\nrefused", "legacy")).toBe("connection\nrefused")
108+
})
109+
110+
test("keeps the historical C1 hash profile explicitly separate", () => {
111+
const raw = "request failed: \x9b31mpassword\x9b0m is connection refused"
112+
const cleaned = Ansi.normalize(raw, "template")!
113+
expect(cleaned).toBe("request failed: password is connection refused")
114+
expect(ErrorDetail.describeToolError(raw).errorMessage).toBe("request failed <x>")
115+
expect(ErrorDetail.describeToolError(raw).errorHash).toBe("40678db384156848")
116+
expect(Telemetry.hashError(Telemetry.maskString(cleaned).slice(0, 500))).toBe("ae7e754cfb2d72a2")
117+
})
118+
})
119+
120+
describe("pre-V4 maskString compatibility", () => {
121+
test.each(cases)("pins the historical error hash for %s", (name, raw) => {
122+
expect(ErrorDetail.describeToolError(raw).errorHash).toBe(hashes[name])
123+
})
124+
125+
test("preserves complete masked output over the frozen 96-input corpus", () => {
126+
expect(corpus.length).toBe(96)
127+
const digest = createHash("sha256")
128+
.update(JSON.stringify(corpus.map((raw) => Telemetry.maskString(raw))))
129+
.digest("hex")
130+
expect(digest).toBe("1fe645e8c04b5f30482bce034f9ddde64a36ac20beb475696a384edb39e1028c")
131+
})
132+
})

‎packages/opencode/test/telemetry/error-detail.test.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,33 @@ describe("templateErrorMessage", () => {
206206
expect(templateErrorMessage("error ".repeat(200)).length).toBeLessThanOrEqual(200)
207207
})
208208

209+
test.each([8191, 8192, 8193])("bounds the raw first line at %i characters", (length) => {
210+
const raw = "connection refused" + " ".repeat(length - 18)
211+
expect(templateErrorMessage(raw)).toBe(length > 8192 ? "<x>" : "connection refused")
212+
})
213+
214+
test.each([
215+
["diagnostic words", "error ".repeat(2000)],
216+
["closed CSI across the limit", "connection \x1b[" + "0;".repeat(5000) + "mpassword=wrong user"],
217+
["closed OSC across the limit", "connection \x1b]0;" + "x".repeat(8192) + "\x07password=wrong user"],
218+
["unfinished CSI", "connection \x1b[" + "0;".repeat(5000)],
219+
["unfinished OSC", "connection \x1b]0;" + "wrong user ".repeat(1000)],
220+
["credential word across the limit", " ".repeat(8190) + "privateKey=wrong user"],
221+
["spaced key across the limit", "connection " + " ".repeat(8180) + "connection string=wrong user"],
222+
["styled key across the limit", " ".repeat(8190) + "pass\x1b[31mword=wrong user"],
223+
])("fails closed for an oversized first line: %s", (_, raw) => {
224+
expect(templateErrorMessage(raw)).toBe("<x>")
225+
})
226+
227+
test.each(["\n", "\r", "\r\n", "\u2028", "\u2029"])(
228+
"keeps a complete line at the input limit before %j",
229+
(newline) => {
230+
const first = "connection refused" + " ".repeat(8192 - 18)
231+
expect(templateErrorMessage(first + newline + "password=wrong user")).toBe("connection refused")
232+
expect(templateErrorMessage("connection refused" + newline + "\x1b[31m".repeat(10000))).toBe("connection refused")
233+
},
234+
)
235+
209236
test.each(["\n", "\r", "\r\n", "\u2028", "\u2029"])("does not inspect later lines after %j", (newline) => {
210237
expect(templateErrorMessage(`connection refused${newline}password is wrong user`)).toBe("connection refused")
211238
expect(templateErrorMessage(`password is wrong user${newline}connection refused`)).toBe("<x>")

0 commit comments

Comments
 (0)