|
| 1 | +import { describe, expect, test } from "bun:test" |
| 2 | +import { Telemetry } from "../../src/altimate/telemetry" |
| 3 | +import { ErrorDetail } from "../../src/altimate/telemetry/error-detail" |
| 4 | + |
| 5 | +const { buildErrorFingerprintEvent, describeToolError, templateErrorMessage } = ErrorDetail |
| 6 | +// built at runtime so no credential-shaped literal is committed |
| 7 | +const basicAuth = Buffer.from(["adm", "pw"].join(":")).toString("base64") |
| 8 | +const pgUri = ["postgres:", "//", "u", ":", "p", "@db:5432/x"].join("") |
| 9 | + |
| 10 | +describe("templateErrorMessage", () => { |
| 11 | + test.each([ |
| 12 | + ["column ssn_number does not exist in table hr.salaries", "column <x> does not exist in table <x>"], |
| 13 | + ["password: hunter2", "password: <x>"], |
| 14 | + ["Cannot delete file: permission denied", "Cannot delete file: permission denied"], |
| 15 | + ["failed (code 12)!", "failed (code <x>)!"], |
| 16 | + ])("%s", (raw, expected) => { |
| 17 | + expect(templateErrorMessage(raw)).toBe(expected) |
| 18 | + }) |
| 19 | + |
| 20 | + test("Basic auth header keeps no secret", () => { |
| 21 | + const out = templateErrorMessage(`Authorization: Basic ${basicAuth}`) |
| 22 | + expect(out).toBe("Authorization: <x>") |
| 23 | + }) |
| 24 | + |
| 25 | + test("unquoted SQL keeps no identifiers, numbers or names", () => { |
| 26 | + const out = templateErrorMessage("SELECT ssn FROM prod.hr.salaries WHERE id = 4417 AND name = Alice") |
| 27 | + for (const leak of ["prod", "hr.salaries", "4417", "Alice"]) expect(out).not.toContain(leak) |
| 28 | + }) |
| 29 | + |
| 30 | + test.each([ |
| 31 | + "/Users/bob/proj/a.ts", |
| 32 | + "C:\\Users\\bob\\a.ts", |
| 33 | + |
| 34 | + "https://h.example.com/p?token=zzz", |
| 35 | + pgUri, |
| 36 | + "AKIAIOSFODNN7EXAMPLE", |
| 37 | + "ghp_abcdefghijklmnopqrstuvwxyz0123456789", |
| 38 | + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.sig", |
| 39 | + "deadbeefdeadbeefdeadbeefdeadbeef0123", |
| 40 | + "'hunter2' \"s3cret\"", |
| 41 | + "stg_salesforce_history_2024.sql", |
| 42 | + ])("replaces identifier-like token in %s", (raw) => { |
| 43 | + const out = templateErrorMessage(`bad ${raw} here`) |
| 44 | + expect(out).toBe("bad <x> here") |
| 45 | + }) |
| 46 | + |
| 47 | + test.each([ |
| 48 | + ["login failed password: supersecret", "login failed password: <x>"], |
| 49 | + ["Authorization: Bearer secrettoken", "Authorization: <x>"], |
| 50 | + ["bad api key hunter, retry", "bad api key <x>, retry"], |
| 51 | + ["client_secret: supersecret", "<x>"], |
| 52 | + ["access_token: secrettoken", "<x>"], |
| 53 | + ["api-key: secretvalue", "<x>"], |
| 54 | + ["password: correct horse battery staple", "password: <x>"], |
| 55 | + ["Alice does not have access", "<x> does not have access"], |
| 56 | + ["failed\raccount alice unavailable", "failed"], |
| 57 | + ["Error: value 'private customer phrase is invalid", "Error: value <x>"], |
| 58 | + ["xKqPzLmNaBcD failed", "<x> failed"], |
| 59 | + ["DROP TABLE prod.customer_records", "<x>"], |
| 60 | + ])("masks alphabetic credential values in %s", (raw, expected) => { |
| 61 | + expect(templateErrorMessage(raw)).toBe(expected) |
| 62 | + }) |
| 63 | + |
| 64 | + test("only the first line, capped at 200 chars", () => { |
| 65 | + expect(templateErrorMessage("first line\nsecond line")).toBe("first line") |
| 66 | + expect(templateErrorMessage("word ".repeat(200)).length).toBeLessThanOrEqual(200) |
| 67 | + }) |
| 68 | +}) |
| 69 | + |
| 70 | +describe("error_fingerprint wiring", () => { |
| 71 | + const raw = "column ssn_number does not exist in table hr.salaries\n at run (/w/packages/opencode/src/a.ts:1:1)" |
| 72 | + |
| 73 | + test("error_hash is identical to the pre-existing computation", () => { |
| 74 | + expect(describeToolError(raw).errorHash).toBe("3a0bc6389647e452") |
| 75 | + }) |
| 76 | + |
| 77 | + test("event carries the template", () => { |
| 78 | + const d = describeToolError(raw) |
| 79 | + const base = { |
| 80 | + ...d, |
| 81 | + errorClass: "unknown", |
| 82 | + toolName: "bash", |
| 83 | + toolCategory: "shell", |
| 84 | + recovered: false, |
| 85 | + recoveryTool: "", |
| 86 | + } |
| 87 | + const ev: any = buildErrorFingerprintEvent("s1", base) |
| 88 | + expect(ev.type).toBe("error_fingerprint") |
| 89 | + expect(ev.error_message).toBe("column <x> does not exist in table <x>") |
| 90 | + expect(ev.error_hash).toBe(d.errorHash) |
| 91 | + expect("stack" in ev).toBe(false) |
| 92 | + }) |
| 93 | + |
| 94 | + test("prompt.ts emits through these helpers", async () => { |
| 95 | + const src = await Bun.file(new URL("../../src/session/prompt.ts", import.meta.url)).text() |
| 96 | + expect(src).toContain("...ErrorDetail.describeToolError(errorMsg)") |
| 97 | + expect(src).toContain("Telemetry.track(ErrorDetail.buildErrorFingerprintEvent(sessionID, err))") |
| 98 | + }) |
| 99 | +}) |
0 commit comments