Skip to content

Commit 58f8ab4

Browse files
fix(web): bare URL links keep trailing underscores and tildes (#27794)
1 parent 7e01b54 commit 58f8ab4

2 files changed

Lines changed: 163 additions & 2 deletions

File tree

‎packages/shared/src/markdownPipeline.test.ts‎

Lines changed: 114 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,120 @@ import { unified } from "unified";
33
import rehypeRaw from "rehype-raw";
44
import remarkParse from "remark-parse";
55
import remarkRehype from "remark-rehype";
6-
import { CHAT_MARKDOWN_REHYPE_PLUGINS, CHAT_MARKDOWN_REMARK_PLUGINS } from "./markdownPipeline.ts";
6+
import type { Root, RootContent } from "mdast";
7+
import {
8+
CHAT_MARKDOWN_REHYPE_PLUGINS,
9+
CHAT_MARKDOWN_REMARK_PLUGINS,
10+
CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS,
11+
} from "./markdownPipeline.ts";
12+
13+
it.each([
14+
["default", CHAT_MARKDOWN_REMARK_PLUGINS],
15+
["with breaks", CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS],
16+
] as const)("preserves trailing underscores and tildes in bare URLs (%s)", (_, plugins) => {
17+
const processor = unified().use(remarkParse).use(plugins);
18+
const shape = (node: Root | RootContent): string => {
19+
if (node.type === "text") return node.value;
20+
if (node.type === "inlineCode") return `\`${node.value}\``;
21+
if (node.type === "code") return `\`\`\`${node.value}\`\`\``;
22+
const children = "children" in node ? node.children.map(shape).join("") : "";
23+
if (node.type === "link") return `[${children}](${node.url})`;
24+
if (node.type === "linkReference") return `[${children}][${node.identifier}]`;
25+
if (node.type === "emphasis") return `*${children}*`;
26+
if (node.type === "strong") return `**${children}**`;
27+
if (node.type === "delete") return `~~${children}~~`;
28+
return children;
29+
};
30+
const link = (url: string) => `[${url}](${/^www\./i.test(url) ? `http://${url}` : url})`;
31+
for (const url of [
32+
"https://x.com/dimasike_",
33+
"http://example.com/name__",
34+
"HTTPS://example.com/name_",
35+
"https://example.com/a_(b)_",
36+
"https://example.com/?name=value_",
37+
"https://example.com/#section_",
38+
"https://example.com/a_?name=value_#section_",
39+
"https://example.com/name~",
40+
"https://example.com/name~~",
41+
"https://example.com/name_~",
42+
"https://example.com/name~_",
43+
"https://example.com/name__~~_",
44+
"https://example.com/?name=value~",
45+
"https://example.com/#section~",
46+
"www.example.com/name_",
47+
"www.example.com/name~_",
48+
"WWW.example.com/name~~",
49+
"https://example.com/name-",
50+
"https://example.com/name/",
51+
"https://example.com/name%5F",
52+
"https://example.com/name%7E",
53+
]) {
54+
for (const punctuation of ["", ".", ",", "!", "?", ":", ";", ")", "]", '"']) {
55+
const markdown = `See ${url}${punctuation} next`;
56+
expect(shape(processor.runSync(processor.parse(markdown), markdown) as Root), markdown).toBe(
57+
`See ${link(url)}${punctuation} next`,
58+
);
59+
}
60+
}
61+
for (const [markdown, expected] of [
62+
["*https://x.com/dimasike_*", `*${link("https://x.com/dimasike_")}*`],
63+
["**https://x.com/dimasike_**", `**${link("https://x.com/dimasike_")}**`],
64+
["_https://x.com/dimasike_", `*${link("https://x.com/dimasike")}*`],
65+
["__https://x.com/dimasike__", `**${link("https://x.com/dimasike")}**`],
66+
["~~https://example.com/name_~~", `~~${link("https://example.com/name_")}~~`],
67+
["~~https://example.com/name~~", `~~${link("https://example.com/name")}~~`],
68+
["~https://example.com/name~", `~~${link("https://example.com/name")}~~`],
69+
["*https://example.com/name~*", `*${link("https://example.com/name~")}*`],
70+
["_https://example.com/name~_", `*${link("https://example.com/name~")}*`],
71+
["[test](https://example.com/name~)", "[test](https://example.com/name~)"],
72+
["[https://x.com/a](https://x.com/a)~_", `${link("https://x.com/a")}~_`],
73+
["<https://x.com/a>~_", `${link("https://x.com/a")}~_`],
74+
["[test][ref]~_\n\n[ref]: https://x.com/a~_", "[test][ref]~_"],
75+
["`https://example.com/name~_`", "`https://example.com/name~_`"],
76+
["```\nhttps://example.com/name~_\n```", "```https://example.com/name~_```"],
77+
["https\\://example.com/name~_", link("https://example.com/name~_")],
78+
["https://example.com/name. _~", `${link("https://example.com/name")}. _~`],
79+
80+
["Mail [email protected] ~_", "Mail [[email protected]](mailto:[email protected]) ~_"],
81+
["[test](https://x.com/dimasike_)", "[test](https://x.com/dimasike_)"],
82+
["[https://x.com/a](https://x.com/a)_", `${link("https://x.com/a")}_`],
83+
["<https://x.com/a>_", `${link("https://x.com/a")}_`],
84+
["[test][ref]_\n\n[ref]: https://x.com/a_", "[test][ref]_"],
85+
["`https://x.com/dimasike_`", "`https://x.com/dimasike_`"],
86+
["```\nhttps://x.com/dimasike_\n```", "```https://x.com/dimasike_```"],
87+
["https://x.com/a. _", `${link("https://x.com/a")}. _`],
88+
["https\\://x.com/dimasike_", link("https://x.com/dimasike_")],
89+
["https://x.com/a\\_", link("https://x.com/a\\_")],
90+
["https://x.com/a&#95;", `${link("https://x.com/a&#95")};`],
91+
[
92+
"https://x.com/a_ https://x.com/b_",
93+
`${link("https://x.com/a_")} ${link("https://x.com/b_")}`,
94+
],
95+
] as const) {
96+
expect(shape(processor.runSync(processor.parse(markdown), markdown) as Root), markdown).toBe(
97+
expected,
98+
);
99+
}
100+
for (const url of ["https://x.com/dimasike_", "www.example.com/name~_"]) {
101+
const markdown = `${url}, next`;
102+
const boundary = { line: 1, column: url.length + 1, offset: url.length };
103+
expect(processor.runSync(processor.parse(markdown), markdown)).toMatchObject({
104+
children: [
105+
{
106+
children: [
107+
{
108+
type: "link",
109+
url: /^www\./i.test(url) ? `http://${url}` : url,
110+
position: { end: boundary },
111+
children: [{ type: "text", value: url, position: { end: boundary } }],
112+
},
113+
{ type: "text", value: ", next", position: { start: boundary } },
114+
],
115+
},
116+
],
117+
});
118+
}
119+
});
7120

8121
it("preserves in-app thread links through the shared Markdown sanitizer", () => {
9122
const processor = unified()

‎packages/shared/src/markdownPipeline.ts‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import type { Paragraph, PhrasingContent, Root, RootContent } from "mdast";
22
import { remarkKeepWindowsPaths } from "./markdownWindowsPaths.ts";
3-
import { type PluggableList, type Processor, unified } from "unified";
3+
import { type PluggableList, type Processor, type Transformer, unified } from "unified";
44
import remarkParse from "remark-parse";
55
import remarkRehype from "remark-rehype";
66
import { find, html } from "property-information";
@@ -204,6 +204,7 @@ const CHAT_MARKDOWN_SANITIZE_SCHEMA = {
204204

205205
export const CHAT_MARKDOWN_REMARK_PLUGINS: PluggableList = [
206206
remarkGfm,
207+
remarkKeepAutolinkSuffixes,
207208
remarkKeepWindowsPaths,
208209
remarkGithubAlerts,
209210
remarkNormalizeListItemIndentation,
@@ -215,6 +216,7 @@ export const CHAT_MARKDOWN_REMARK_PLUGINS: PluggableList = [
215216

216217
export const CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS: PluggableList = [
217218
remarkGfm,
219+
remarkKeepAutolinkSuffixes,
218220
remarkKeepWindowsPaths,
219221
remarkGithubAlerts,
220222
remarkNormalizeListItemIndentation,
@@ -368,6 +370,52 @@ type MarkdownAstNode = {
368370
children?: MarkdownAstNode[];
369371
};
370372

373+
function remarkKeepAutolinkSuffixes(): Transformer<Root> {
374+
return (tree, file) => {
375+
const source = String(file);
376+
const visit = (node: Root | RootContent) => {
377+
if (!("children" in node) || node.type === "link" || node.type === "linkReference") return;
378+
for (let index = 0; index < node.children.length; index++) {
379+
const child = node.children[index];
380+
const next = node.children[index + 1];
381+
if (child?.type === "link" && /^https?:\/\//i.test(child.url) && next?.type === "text") {
382+
const suffix = /^[_~]+/.exec(next.value)?.[0];
383+
if (!suffix) continue;
384+
const label = child.children.length === 1 ? child.children[0] : undefined;
385+
if (label?.type !== "text") continue;
386+
if (label.value !== child.url && `http://${label.value}` !== child.url) continue;
387+
const start = child.position?.start.offset;
388+
const end = child.position?.end.offset;
389+
const bare =
390+
!child.position ||
391+
(start !== undefined &&
392+
end !== undefined &&
393+
source.slice(start, end) === label.value &&
394+
source.slice(end, end + suffix.length) === suffix);
395+
if (bare) {
396+
child.url += suffix;
397+
label.value += suffix;
398+
next.value = next.value.slice(suffix.length);
399+
if (child.position && end !== undefined) {
400+
const boundary = {
401+
...child.position.end,
402+
column: child.position.end.column + suffix.length,
403+
offset: end + suffix.length,
404+
};
405+
child.position.end = boundary;
406+
if (label.position) label.position.end = boundary;
407+
if (next.position) next.position.start = boundary;
408+
}
409+
if (!next.value) node.children.splice(index + 1, 1);
410+
}
411+
}
412+
if (child) visit(child);
413+
}
414+
};
415+
visit(tree);
416+
};
417+
}
418+
371419
function remarkPreserveCodeMeta() {
372420
return (tree: MarkdownAstNode) => {
373421
const visit = (node: MarkdownAstNode) => {

0 commit comments

Comments
 (0)