Skip to content

Commit 2a778f7

Browse files
scratchyoneclaudeYash-Singh1
authored
feat(web): shell commands in the timeline are syntax highlighted (pingdotgg#15037)
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]> Co-authored-by: Yash Singh <[email protected]>
1 parent c408737 commit 2a778f7

12 files changed

Lines changed: 1266 additions & 61 deletions

‎apps/web/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060
"remark-breaks": "^4.0.0",
6161
"remark-gfm": "^4.0.1",
6262
"tailwind-merge": "^3.4.0",
63+
"unbash": "^5.0.0",
6364
"zustand": "^5.0.11"
6465
},
6566
"devDependencies": {
Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
import { describe, expect, it } from "vite-plus/test";
2+
3+
import { embeddedScripts } from "../../lib/embeddedScripts";
4+
import { keyedLines, withEmbeddedScripts, wordsOf } from "./HighlightedTokens";
5+
6+
function tokensOf(...contents: string[]) {
7+
let offset = 0;
8+
return contents.map((content) => {
9+
const token = { content, offset };
10+
offset += content.length;
11+
return token;
12+
});
13+
}
14+
15+
function shape(parts: ReturnType<typeof wordsOf>) {
16+
return parts.map((part) =>
17+
typeof part === "string" ? part : part.pieces.map((piece) => piece.text).join(""),
18+
);
19+
}
20+
21+
describe("wordsOf", () => {
22+
it.each([
23+
// Shiki splits these shell words into several tokens.
24+
[["echo", " ", '"', "$HOME", "/${", "PATH", "%%:*", '}"'], 'echo "$HOME/${PATH%%:*}"'],
25+
[["rsync", " ", "-a", " ", "--exclude", " ", "build"], "rsync -a --exclude build"],
26+
// A token can carry whitespace on either side or inside a string.
27+
[["git", " && ", "echo", " ", '"two words"', "; "], 'git && echo "two words"; '],
28+
[[" ", "indented"], " indented"],
29+
[[], ""],
30+
])("wraps the same words as the plain-text fallback: %j", (contents, line) => {
31+
expect(shape(wordsOf(tokensOf(...contents)))).toEqual(
32+
line.split(/(\s+)/u).filter((part) => part !== ""),
33+
);
34+
});
35+
36+
it("keeps each piece's own token so colors survive regrouping", () => {
37+
const [word] = wordsOf(tokensOf('"', "$HOME", '"'));
38+
expect(
39+
typeof word === "string" ? null : word?.pieces.map((piece) => piece.token.content),
40+
).toEqual(['"', "$HOME", '"']);
41+
});
42+
});
43+
44+
describe("keyedLines", () => {
45+
it.each([
46+
"git status\ngit diff",
47+
"Get-ChildItem\r\nWrite-Output done\r\n",
48+
"mixed\r\nendings\n\nlone \r stays",
49+
])("renders the same text as the source: %j", (code) => {
50+
// Shiki splits lines on \r?\n and drops the ending from the tokens.
51+
const lines = code.split(/\r?\n/u).map((line) => tokensOf(line));
52+
const rendered = keyedLines(code, lines)
53+
.map(({ tokens, ending }) => tokens.map((token) => token.content).join("") + ending)
54+
.join("");
55+
expect(rendered).toBe(code);
56+
});
57+
58+
it("keys empty lines apart", () => {
59+
const code = "a\r\n\r\n\r\nb";
60+
const keys = keyedLines(
61+
code,
62+
code.split(/\r?\n/u).map((line) => tokensOf(line)),
63+
).map((line) => line.key);
64+
expect(new Set(keys).size).toBe(keys.length);
65+
});
66+
});
67+
68+
describe("withEmbeddedScripts", () => {
69+
// A stand-in grammar: each word is one token colored with the language name,
70+
// and lines split the way Shiki splits them.
71+
function tokenize(text: string, language: string) {
72+
if (language === "unknown") throw new Error("Language not found");
73+
let lineStart = 0;
74+
return text.split(/\r?\n/u).map((line) => {
75+
const tokens = [...line.matchAll(/\S+|\s+/gu)].map((match) => ({
76+
content: match[0],
77+
offset: lineStart + match.index,
78+
color: language,
79+
}));
80+
lineStart +=
81+
line.length + (text.slice(lineStart + line.length).match(/^\r?\n/u)?.[0].length ?? 0);
82+
return tokens;
83+
});
84+
}
85+
86+
function colorsOf(code: string, scripts = embeddedScripts(code)) {
87+
const lines = withEmbeddedScripts(code, tokenize(code, "shellscript"), scripts, tokenize);
88+
expect(lines.map((line) => line.map((token) => token.content).join("")).join("\n")).toBe(
89+
code.replaceAll("\r\n", "\n"),
90+
);
91+
return lines.flat().map((token) => [token.content, token.color]);
92+
}
93+
94+
it("colors each nested script with its own grammar, keeping the text", () => {
95+
expect(colorsOf(`bash -lc "python3 -c 'print(1)'"`)).toEqual([
96+
[`bash -lc "python3 -c '`, "shellscript"],
97+
["print(1)", "python"],
98+
[`'"`, "shellscript"],
99+
]);
100+
});
101+
102+
it("colors an escape sequence like the character it spells", () => {
103+
expect(colorsOf(`bash -c "python3 -c 'print(\\"x\\")'"`)).toContainEqual([
104+
`print(\\"x\\")`,
105+
"python",
106+
]);
107+
});
108+
109+
it("keeps the surrounding colors where a grammar is missing", () => {
110+
const code = "cat > x.ts <<'EOF'\r\nbody\r\nEOF";
111+
const [script] = embeddedScripts(code);
112+
// Compare per character: untouched lines keep their grammar's token boundaries.
113+
const characterColors = (scripts: ReturnType<typeof embeddedScripts>) =>
114+
colorsOf(code, scripts).flatMap(([content, color]) => [...content!].map(() => color));
115+
expect(characterColors([{ ...script!, language: "unknown" }])).toEqual(characterColors([]));
116+
});
117+
});
Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
import { use, useMemo, type CSSProperties } from "react";
2+
3+
import { resolveDiffThemeName } from "../../lib/diffRendering";
4+
import type { EmbeddedScript } from "../../lib/embeddedScripts";
5+
import { getSyntaxHighlighterPromise } from "../../lib/syntaxHighlighting";
6+
7+
interface SyntaxToken {
8+
readonly content: string;
9+
readonly offset: number;
10+
readonly color?: string;
11+
readonly fontStyle?: number;
12+
}
13+
14+
type TokenLines = ReadonlyArray<ReadonlyArray<SyntaxToken>>;
15+
16+
const NO_EMBEDDED_SCRIPTS: ReadonlyArray<EmbeddedScript> = [];
17+
18+
function syntaxTokenStyle(token: SyntaxToken): CSSProperties {
19+
const fontStyle = token.fontStyle ?? 0;
20+
return {
21+
...(token.color ? { color: token.color } : {}),
22+
...(fontStyle & 1 ? { fontStyle: "italic" } : {}),
23+
...(fontStyle & 2 ? { fontWeight: 700 } : {}),
24+
...(fontStyle & 4 ? { textDecoration: "underline" } : {}),
25+
};
26+
}
27+
28+
/**
29+
* Colors `code` inside the caller's `<pre>` without changing its text, so
30+
* selection and copy match the plain version. Suspends while the grammar
31+
* loads; wrap it in Suspense with the plain text as the fallback.
32+
* `wordClassName` goes on each whitespace-separated word. `embedded`
33+
* scripts are colored with their own grammar inside `code`.
34+
*/
35+
export function HighlightedTokens({
36+
code,
37+
language,
38+
embedded = NO_EMBEDDED_SCRIPTS,
39+
theme,
40+
wordClassName,
41+
}: {
42+
code: string;
43+
language: string;
44+
embedded?: ReadonlyArray<EmbeddedScript>;
45+
theme: "light" | "dark";
46+
wordClassName?: string;
47+
}) {
48+
const highlighter = use(getSyntaxHighlighterPromise(language));
49+
// Every grammar loads into the same shared highlighter.
50+
for (const script of embedded) use(getSyntaxHighlighterPromise(script.language));
51+
const lines = useMemo(() => {
52+
const themeName = resolveDiffThemeName(theme);
53+
const tokenize = (text: string, lang: string) =>
54+
highlighter.codeToTokens(text, { lang, theme: themeName }).tokens;
55+
return keyedLines(
56+
code,
57+
withEmbeddedScripts(code, tokenize(code, language), embedded, tokenize),
58+
);
59+
}, [code, embedded, highlighter, language, theme]);
60+
61+
return lines.map(({ key, tokens, ending }) => (
62+
<span key={key}>
63+
{wordClassName
64+
? wordsOf(tokens).map((part) =>
65+
typeof part === "string" ? (
66+
part
67+
) : (
68+
<span key={part.key} className={wordClassName}>
69+
{part.pieces.map((piece) => (
70+
<span key={piece.key} style={syntaxTokenStyle(piece.token)}>
71+
{piece.text}
72+
</span>
73+
))}
74+
</span>
75+
),
76+
)
77+
: tokens.map((token) => (
78+
<span key={`${token.offset}:${token.content}`} style={syntaxTokenStyle(token)}>
79+
{token.content}
80+
</span>
81+
))}
82+
{ending}
83+
</span>
84+
));
85+
}
86+
87+
/**
88+
* Recolors each embedded script's span of `code` with its own grammar,
89+
* outermost first so nested scripts win. A script whose grammar is missing
90+
* keeps the colors around it.
91+
*/
92+
export function withEmbeddedScripts(
93+
code: string,
94+
lines: TokenLines,
95+
embedded: ReadonlyArray<EmbeddedScript>,
96+
tokenize: (text: string, language: string) => TokenLines,
97+
): TokenLines {
98+
if (embedded.length === 0) return lines;
99+
const styles = Array.from<SyntaxToken | undefined>({ length: code.length });
100+
for (const token of lines.flat()) {
101+
styles.fill(token, token.offset, token.offset + token.content.length);
102+
}
103+
for (const script of embedded) {
104+
let scriptLines: TokenLines;
105+
try {
106+
scriptLines = tokenize(script.text, script.language);
107+
} catch {
108+
continue;
109+
}
110+
for (const token of scriptLines.flat()) {
111+
if (token.content === "") continue;
112+
const last = token.offset + token.content.length - 1;
113+
styles.fill(token, script.starts[token.offset], script.ends[last]);
114+
}
115+
}
116+
return restyledLines(code, styles);
117+
}
118+
119+
/** Splits `code` into lines of tokens, one per run of characters sharing a style. */
120+
function restyledLines(
121+
code: string,
122+
styles: ReadonlyArray<SyntaxToken | undefined>,
123+
): SyntaxToken[][] {
124+
const lines: SyntaxToken[][] = [];
125+
let start = 0;
126+
for (const lineBreak of [...code.matchAll(/\r?\n/gu), undefined]) {
127+
const end = lineBreak?.index ?? code.length;
128+
const tokens: SyntaxToken[] = [];
129+
for (let index = start; index < end;) {
130+
const style = styles[index];
131+
let next = index + 1;
132+
while (
133+
next < end &&
134+
styles[next]?.color === style?.color &&
135+
styles[next]?.fontStyle === style?.fontStyle
136+
) {
137+
next += 1;
138+
}
139+
tokens.push({
140+
content: code.slice(index, next),
141+
offset: index,
142+
...(style?.color ? { color: style.color } : {}),
143+
...(style?.fontStyle ? { fontStyle: style.fontStyle } : {}),
144+
});
145+
index = next;
146+
}
147+
lines.push(tokens);
148+
if (lineBreak) start = lineBreak.index + lineBreak[0].length;
149+
}
150+
return lines;
151+
}
152+
153+
/**
154+
* Keys each line by its start offset, which stays unique when several lines
155+
* are empty, and pairs it with the line ending `code` has after it. Shiki drops
156+
* `\n` and `\r\n` from tokens alike, so CRLF has to come back from the source.
157+
*/
158+
export function keyedLines<Token extends SyntaxToken>(
159+
code: string,
160+
lines: ReadonlyArray<ReadonlyArray<Token>>,
161+
) {
162+
const endings = code.match(/\r?\n/gu) ?? [];
163+
let lineStart = 0;
164+
return lines.map((tokens, index) => {
165+
const text = tokens.map((token) => token.content).join("");
166+
const key = `${lineStart}:${text}`;
167+
const ending = endings[index] ?? "";
168+
lineStart += text.length + ending.length;
169+
return { key, tokens, ending };
170+
});
171+
}
172+
173+
interface Word {
174+
readonly key: string;
175+
readonly pieces: Array<{ key: string; text: string; token: SyntaxToken }>;
176+
}
177+
178+
/**
179+
* Regroups a line's tokens into whitespace runs and whole words, matching
180+
* `text.split(/(\s+)/)`, so a word made of several tokens (`"$HOME/x"`) wraps
181+
* exactly like its plain-text fallback.
182+
*/
183+
export function wordsOf(tokens: ReadonlyArray<SyntaxToken>): Array<string | Word> {
184+
const parts: Array<string | Word> = [];
185+
let word: Word | null = null;
186+
for (const token of tokens) {
187+
let offset = token.offset;
188+
for (const text of token.content.split(/(\s+)/u)) {
189+
if (text === "") continue;
190+
if (/^\s/u.test(text)) {
191+
parts.push(text);
192+
word = null;
193+
} else {
194+
if (!word) {
195+
word = { key: String(offset), pieces: [] };
196+
parts.push(word);
197+
}
198+
word.pieces.push({ key: String(offset), text, token });
199+
}
200+
offset += text.length;
201+
}
202+
}
203+
return parts;
204+
}

‎apps/web/src/components/chat/MessagesTimeline.logic.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ export function workEntryDisplayLabel(entry: WorkLogEntry, workspaceRoot: string
121121
}
122122
const toolPresentation = resolveWorkEntryToolPresentation(entry);
123123
if (toolPresentation) return toolPresentation.displayName;
124-
if (entry.command) return commandDisplayText(entry.command);
124+
if (entry.command?.trim()) return commandDisplayText(entry.command);
125125
const action = toolGroupAction(entry);
126126
if (action === "code-search" || action === "search") {
127127
// Adapters title file searches with their target; the item keeps only the pattern.

0 commit comments

Comments
 (0)