Skip to content

Commit ec4f1c1

Browse files
scratchyoneclaude
andcommitted
feat(web): scripts inside commands get their own highlighting
The expanded block showed Codex rows without their `/bin/zsh -lc "…"` wrapper, and code inside a command, like the Python in a `python3 - <<'PY'` heredoc, was colored as one shell string. The block now shows the command exactly as it ran, and colors each script inside it with its own grammar: a `bash -c` script as shell, `python3 -c`, `node -e`, `sqlite3 db "…"` and similar arguments in their language, a heredoc in the language of the interpreter reading it, the file `cat` or `tee` writes, or a delimiter like `<<'SQL'`. Scripts nest, so a Python heredoc inside a Codex wrapper is colored as Python. Escapes map back to the characters that spell them. Commands are parsed with unbash, loaded with the first expanded command. A command that does not parse cleanly, such as one Codex's secret redaction broke, keeps plain shell coloring. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent a5fc315 commit ec4f1c1

8 files changed

Lines changed: 765 additions & 22 deletions

File tree

‎apps/web/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@
5656
"remark-breaks": "^4.0.0",
5757
"remark-gfm": "^4.0.1",
5858
"tailwind-merge": "^3.4.0",
59+
"unbash": "^5.0.0",
5960
"zustand": "^5.0.11"
6061
},
6162
"devDependencies": {

‎apps/web/src/components/chat/HighlightedTokens.test.ts‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { describe, expect, it } from "vite-plus/test";
22

3-
import { keyedLines, wordsOf } from "./HighlightedTokens";
3+
import { embeddedScripts } from "../../lib/embeddedScripts";
4+
import { keyedLines, withEmbeddedScripts, wordsOf } from "./HighlightedTokens";
45

56
function tokensOf(...contents: string[]) {
67
let offset = 0;
@@ -63,3 +64,54 @@ describe("keyedLines", () => {
6364
expect(new Set(keys).size).toBe(keys.length);
6465
});
6566
});
67+
68+
describe("withEmbeddedScripts", () => {
69+
// A stand-in grammar: each word is one token colored with the language name,
70+
// and lines split the way Shiki splits them.
71+
function tokenize(text: string, language: string) {
72+
if (language === "unknown") throw new Error("Language not found");
73+
let lineStart = 0;
74+
return text.split(/\r?\n/u).map((line) => {
75+
const tokens = [...line.matchAll(/\S+|\s+/gu)].map((match) => ({
76+
content: match[0],
77+
offset: lineStart + match.index,
78+
color: language,
79+
}));
80+
lineStart +=
81+
line.length + (text.slice(lineStart + line.length).match(/^\r?\n/u)?.[0].length ?? 0);
82+
return tokens;
83+
});
84+
}
85+
86+
function colorsOf(code: string, scripts = embeddedScripts(code)) {
87+
const lines = withEmbeddedScripts(code, tokenize(code, "shellscript"), scripts, tokenize);
88+
expect(lines.map((line) => line.map((token) => token.content).join("")).join("\n")).toBe(
89+
code.replaceAll("\r\n", "\n"),
90+
);
91+
return lines.flat().map((token) => [token.content, token.color]);
92+
}
93+
94+
it("colors each nested script with its own grammar, keeping the text", () => {
95+
expect(colorsOf(`bash -lc "python3 -c 'print(1)'"`)).toEqual([
96+
[`bash -lc "python3 -c '`, "shellscript"],
97+
["print(1)", "python"],
98+
[`'"`, "shellscript"],
99+
]);
100+
});
101+
102+
it("colors an escape sequence like the character it spells", () => {
103+
expect(colorsOf(`bash -c "python3 -c 'print(\\"x\\")'"`)).toContainEqual([
104+
`print(\\"x\\")`,
105+
"python",
106+
]);
107+
});
108+
109+
it("keeps the surrounding colors where a grammar is missing", () => {
110+
const code = "cat > x.ts <<'EOF'\r\nbody\r\nEOF";
111+
const [script] = embeddedScripts(code);
112+
// Compare per character: untouched lines keep their grammar's token boundaries.
113+
const characterColors = (scripts: ReturnType<typeof embeddedScripts>) =>
114+
colorsOf(code, scripts).flatMap(([content, color]) => [...content!].map(() => color));
115+
expect(characterColors([{ ...script!, language: "unknown" }])).toEqual(characterColors([]));
116+
});
117+
});

‎apps/web/src/components/chat/HighlightedTokens.tsx‎

Lines changed: 86 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { use, useMemo, type CSSProperties } from "react";
22

33
import { resolveDiffThemeName } from "../../lib/diffRendering";
4+
import type { EmbeddedScript } from "../../lib/embeddedScripts";
45
import { getSyntaxHighlighterPromise } from "../../lib/syntaxHighlighting";
56

67
interface SyntaxToken {
@@ -10,6 +11,10 @@ interface SyntaxToken {
1011
readonly fontStyle?: number;
1112
}
1213

14+
type TokenLines = ReadonlyArray<ReadonlyArray<SyntaxToken>>;
15+
16+
const NO_EMBEDDED_SCRIPTS: ReadonlyArray<EmbeddedScript> = [];
17+
1318
function syntaxTokenStyle(token: SyntaxToken): CSSProperties {
1419
const fontStyle = token.fontStyle ?? 0;
1520
return {
@@ -24,29 +29,34 @@ function syntaxTokenStyle(token: SyntaxToken): CSSProperties {
2429
* Colors `code` inside the caller's `<pre>` without changing its text, so
2530
* selection and copy match the plain version. Suspends while the grammar
2631
* loads; wrap it in Suspense with the plain text as the fallback.
27-
* `wordClassName` goes on each whitespace-separated word.
32+
* `wordClassName` goes on each whitespace-separated word. `embedded`
33+
* scripts are colored with their own grammar inside `code`.
2834
*/
2935
export function HighlightedTokens({
3036
code,
3137
language,
38+
embedded = NO_EMBEDDED_SCRIPTS,
3239
theme,
3340
wordClassName,
3441
}: {
3542
code: string;
3643
language: string;
44+
embedded?: ReadonlyArray<EmbeddedScript>;
3745
theme: "light" | "dark";
3846
wordClassName?: string;
3947
}) {
4048
const highlighter = use(getSyntaxHighlighterPromise(language));
41-
const lines = useMemo(
42-
() =>
43-
keyedLines(
44-
code,
45-
highlighter.codeToTokens(code, { lang: language, theme: resolveDiffThemeName(theme) })
46-
.tokens,
47-
),
48-
[code, highlighter, language, theme],
49-
);
49+
// Every grammar loads into the same shared highlighter.
50+
for (const script of embedded) use(getSyntaxHighlighterPromise(script.language));
51+
const lines = useMemo(() => {
52+
const themeName = resolveDiffThemeName(theme);
53+
const tokenize = (text: string, lang: string) =>
54+
highlighter.codeToTokens(text, { lang, theme: themeName }).tokens;
55+
return keyedLines(
56+
code,
57+
withEmbeddedScripts(code, tokenize(code, language), embedded, tokenize),
58+
);
59+
}, [code, embedded, highlighter, language, theme]);
5060

5161
return lines.map(({ key, tokens, ending }) => (
5262
<span key={key}>
@@ -74,6 +84,72 @@ export function HighlightedTokens({
7484
));
7585
}
7686

87+
/**
88+
* Recolors each embedded script's span of `code` with its own grammar,
89+
* outermost first so nested scripts win. A script whose grammar is missing
90+
* keeps the colors around it.
91+
*/
92+
export function withEmbeddedScripts(
93+
code: string,
94+
lines: TokenLines,
95+
embedded: ReadonlyArray<EmbeddedScript>,
96+
tokenize: (text: string, language: string) => TokenLines,
97+
): TokenLines {
98+
if (embedded.length === 0) return lines;
99+
const styles = Array.from<SyntaxToken | undefined>({ length: code.length });
100+
for (const token of lines.flat()) {
101+
styles.fill(token, token.offset, token.offset + token.content.length);
102+
}
103+
for (const script of embedded) {
104+
let scriptLines: TokenLines;
105+
try {
106+
scriptLines = tokenize(script.text, script.language);
107+
} catch {
108+
continue;
109+
}
110+
for (const token of scriptLines.flat()) {
111+
if (token.content === "") continue;
112+
const last = token.offset + token.content.length - 1;
113+
styles.fill(token, script.starts[token.offset], script.ends[last]);
114+
}
115+
}
116+
return restyledLines(code, styles);
117+
}
118+
119+
/** Splits `code` into lines of tokens, one per run of characters sharing a style. */
120+
function restyledLines(
121+
code: string,
122+
styles: ReadonlyArray<SyntaxToken | undefined>,
123+
): SyntaxToken[][] {
124+
const lines: SyntaxToken[][] = [];
125+
let start = 0;
126+
for (const lineBreak of [...code.matchAll(/\r?\n/gu), undefined]) {
127+
const end = lineBreak?.index ?? code.length;
128+
const tokens: SyntaxToken[] = [];
129+
for (let index = start; index < end;) {
130+
const style = styles[index];
131+
let next = index + 1;
132+
while (
133+
next < end &&
134+
styles[next]?.color === style?.color &&
135+
styles[next]?.fontStyle === style?.fontStyle
136+
) {
137+
next += 1;
138+
}
139+
tokens.push({
140+
content: code.slice(index, next),
141+
offset: index,
142+
...(style?.color ? { color: style.color } : {}),
143+
...(style?.fontStyle ? { fontStyle: style.fontStyle } : {}),
144+
});
145+
index = next;
146+
}
147+
lines.push(tokens);
148+
if (lineBreak) start = lineBreak.index + lineBreak[0].length;
149+
}
150+
return lines;
151+
}
152+
77153
/**
78154
* Keys each line by its start offset, which stays unique when several lines
79155
* are empty, and pairs it with the line ending `code` has after it. Shiki drops

‎apps/web/src/components/chat/ShellCommandBlock.tsx‎

Lines changed: 34 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import {
2-
commandDisplayText,
32
commandHighlightLanguage,
3+
withVisibleControlCharacters,
44
} from "@t3tools/client-runtime/work-log/command-label";
5-
import { Suspense } from "react";
5+
import { Suspense, use, useMemo } from "react";
66

77
import { useTheme } from "../../hooks/useTheme";
88
import { RenderErrorBoundary } from "../RenderErrorBoundary";
@@ -12,6 +12,13 @@ import { HighlightedTokens } from "./HighlightedTokens";
1212
// at a hyphen; a word longer than the line still breaks anywhere.
1313
const WORD_CLASS_NAME = "inline-block max-w-full [overflow-wrap:anywhere]";
1414

15+
// The shell parser loads with the first expanded command, not with the timeline.
16+
let embeddedScriptsModule: Promise<typeof import("../../lib/embeddedScripts")> | undefined;
17+
function loadEmbeddedScripts() {
18+
embeddedScriptsModule ??= import("../../lib/embeddedScripts");
19+
return embeddedScriptsModule;
20+
}
21+
1522
/** Same layout as the highlighted version, so the grammar arriving never reflows the block. */
1623
function PlainWords({ code }: { code: string }) {
1724
// split with a capture group alternates words (even) and whitespace (odd).
@@ -28,22 +35,39 @@ function PlainWords({ code }: { code: string }) {
2835
);
2936
}
3037

31-
/** The command a command_execution item ran, without a `bash -lc` wrapper, syntax highlighted. */
38+
function HighlightedCommand({ code, theme }: { code: string; theme: "light" | "dark" }) {
39+
const { embeddedScripts } = use(loadEmbeddedScripts());
40+
const language = commandHighlightLanguage(code);
41+
// Only shell syntax nests scripts; PowerShell's own grammar colors its strings.
42+
const embedded = useMemo(
43+
() => (language === "shellscript" ? embeddedScripts(code) : []),
44+
[code, embeddedScripts, language],
45+
);
46+
return (
47+
<HighlightedTokens
48+
code={code}
49+
language={language}
50+
embedded={embedded}
51+
theme={theme}
52+
wordClassName={WORD_CLASS_NAME}
53+
/>
54+
);
55+
}
56+
57+
/**
58+
* The command a command_execution item ran, syntax highlighted. Scripts inside
59+
* it, such as a `bash -lc` script or a Python heredoc, get their own grammar.
60+
*/
3261
export function ShellCommandBlock({ command }: { command: string }) {
3362
const { resolvedTheme } = useTheme();
34-
const code = commandDisplayText(command);
63+
const code = withVisibleControlCharacters(command.trim());
3564
if (!code) return null;
3665
const plain = <PlainWords code={code} />;
3766
return (
3867
<pre className="max-h-80 overflow-auto whitespace-pre-wrap rounded-md border border-border/50 bg-background/60 p-2 font-mono text-2xs leading-relaxed text-foreground/85 select-text">
3968
<RenderErrorBoundary fallback={plain} resetKeys={[code]}>
4069
<Suspense fallback={plain}>
41-
<HighlightedTokens
42-
code={code}
43-
language={commandHighlightLanguage(code)}
44-
theme={resolvedTheme}
45-
wordClassName={WORD_CLASS_NAME}
46-
/>
70+
<HighlightedCommand code={code} theme={resolvedTheme} />
4771
</Suspense>
4872
</RenderErrorBoundary>
4973
</pre>
Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
import { describe, expect, it } from "vite-plus/test";
2+
3+
import { embeddedScripts } from "./embeddedScripts";
4+
5+
/** Each script's language, its text, and the part of the command that spells it. */
6+
function scriptsOf(command: string) {
7+
return embeddedScripts(command).map((script) => {
8+
// Every character maps inside the command, in order.
9+
script.starts.forEach((start, index) => {
10+
expect(start).toBeLessThan(script.ends[index]!);
11+
if (index > 0) expect(start).toBeGreaterThanOrEqual(script.ends[index - 1]!);
12+
});
13+
return [
14+
script.language,
15+
script.text,
16+
command.slice(script.starts[0], script.ends.at(-1)),
17+
] as const;
18+
});
19+
}
20+
21+
const pythonScript = `from pathlib import Path
22+
import json
23+
24+
# print every package.json name and version
25+
for p in sorted(Path('.').glob('**/package.json')):
26+
data = json.loads(p.read_text())
27+
print(f"{data['name']:16} {data.get('version', '?')}")
28+
`;
29+
30+
describe("embeddedScripts", () => {
31+
it("finds the script inside a Codex login-shell wrapper and the script inside that", () => {
32+
// Recorded in the turn_interrupt_mid_tool Codex transcript
33+
// (apps/server/src/orchestration-v2/testkit/fixtures).
34+
const inner = `console.log('interrupt fixture tool started'); setTimeout(() => {}, 30000)`;
35+
const command = `/bin/bash -lc "node -e \\"${inner}\\""`;
36+
expect(scriptsOf(command)).toEqual([
37+
["shellscript", `node -e "${inner}"`, `node -e \\"${inner}\\"`],
38+
["javascript", inner, inner],
39+
]);
40+
});
41+
42+
it("highlights a heredoc fed to an interpreter in that interpreter's language", () => {
43+
const command = `python3 - <<'PY'\n${pythonScript}PY`;
44+
expect(scriptsOf(command)).toEqual([["python", pythonScript, pythonScript]]);
45+
});
46+
47+
it("maps escaped characters back to both source characters", () => {
48+
const command = `/bin/zsh -lc "python3 - <<'PY'\nprint(\\"\\$HOME\\")\nPY"`;
49+
const [shell, python] = scriptsOf(command);
50+
expect(shell?.[1]).toBe(`python3 - <<'PY'\nprint("$HOME")\nPY`);
51+
expect(python).toEqual(["python", `print("$HOME")\n`, `print(\\"\\$HOME\\")\n`]);
52+
});
53+
54+
it.each([
55+
[`python3 -c 'print(1)'`, "python", "print(1)"],
56+
[`python3 -X dev -c 'print(1)'`, "python", "print(1)"],
57+
[`node --eval "process.exit(0)"`, "javascript", "process.exit(0)"],
58+
[`bun -e 'console.log(Bun.version)'`, "typescript", "console.log(Bun.version)"],
59+
[`deno eval 'console.log(1)'`, "typescript", "console.log(1)"],
60+
[`ruby -e 'puts 1'`, "ruby", "puts 1"],
61+
[`perl -E 'say 1'`, "perl", "say 1"],
62+
[`sqlite3 app.db "select * from users"`, "sql", "select * from users"],
63+
[`psql -d app -c 'select 1'`, "sql", "select 1"],
64+
[`pwsh -NoProfile -Command "Get-ChildItem"`, "powershell", "Get-ChildItem"],
65+
[`env FOO=1 sudo -u bob python3 -c 'print(1)'`, "python", "print(1)"],
66+
[`echo "$(node -e 'console.log(1)')"`, "javascript", "console.log(1)"],
67+
[`python3 <<< 'print(1)'`, "python", "print(1)"],
68+
])("finds the inline script in %s", (command, language, script) => {
69+
expect(scriptsOf(command)).toEqual([[language, script, expect.any(String)]]);
70+
});
71+
72+
it.each([
73+
["the file `cat` writes", `cat > src/app.ts <<'EOF'\nconst a = 1;\nEOF`, "typescript"],
74+
["the file `tee` writes", `tee -a config.json <<EOF\n{}\nEOF`, "json"],
75+
["its delimiter", `kubectl apply -f - <<'YAML'\nkind: Pod\nYAML`, "yaml"],
76+
["the shell reading it", `bash <<'EOF'\necho hi\nEOF`, "shellscript"],
77+
])("takes a heredoc's language from %s", (_source, command, language) => {
78+
expect(scriptsOf(command).map(([lang]) => lang)).toEqual([language]);
79+
});
80+
81+
it.each([
82+
["a script file", `python3 build.py <<EOF\ndata\nEOF`],
83+
["a module", `python3 -m json.tool <<EOF\n{}\nEOF`],
84+
["plain commands", `rg -n "foo" src && git status`],
85+
["an unnamed heredoc into a plain command", `git commit -F - <<'EOF'\nfix: x\nEOF`],
86+
["an empty script", `bash -lc ' '`],
87+
])("finds nothing for %s", (_case, command) => {
88+
expect(scriptsOf(command)).toEqual([]);
89+
});
90+
91+
it("trusts nothing in a command whose quoting does not parse", () => {
92+
// Codex's secret redaction dropped the backslash of an escaped quote, so
93+
// the -lc argument ends early and the rest no longer balances.
94+
const command = `/bin/zsh -lc "python3 -c 'token=[REDACTED_SECRET]"apiToken\\"]; print(token)'"`;
95+
expect(scriptsOf(command)).toEqual([]);
96+
});
97+
});

0 commit comments

Comments
 (0)