@@ -3,7 +3,120 @@ import { unified } from "unified";
33import rehypeRaw from "rehype-raw" ;
44import remarkParse from "remark-parse" ;
55import remarkRehype from "remark-rehype" ;
6- import { CHAT_MARKDOWN_REHYPE_PLUGINS , CHAT_MARKDOWN_REMARK_PLUGINS } from "./markdownPipeline.ts" ;
6+ import type { Root , RootContent } from "mdast" ;
7+ import {
8+ CHAT_MARKDOWN_REHYPE_PLUGINS ,
9+ CHAT_MARKDOWN_REMARK_PLUGINS ,
10+ CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS ,
11+ } from "./markdownPipeline.ts" ;
12+
13+ it . each ( [
14+ [ "default" , CHAT_MARKDOWN_REMARK_PLUGINS ] ,
15+ [ "with breaks" , CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS ] ,
16+ ] as const ) ( "preserves trailing underscores and tildes in bare URLs (%s)" , ( _ , plugins ) => {
17+ const processor = unified ( ) . use ( remarkParse ) . use ( plugins ) ;
18+ const shape = ( node : Root | RootContent ) : string => {
19+ if ( node . type === "text" ) return node . value ;
20+ if ( node . type === "inlineCode" ) return `\`${ node . value } \`` ;
21+ if ( node . type === "code" ) return `\`\`\`${ node . value } \`\`\`` ;
22+ const children = "children" in node ? node . children . map ( shape ) . join ( "" ) : "" ;
23+ if ( node . type === "link" ) return `[${ children } ](${ node . url } )` ;
24+ if ( node . type === "linkReference" ) return `[${ children } ][${ node . identifier } ]` ;
25+ if ( node . type === "emphasis" ) return `*${ children } *` ;
26+ if ( node . type === "strong" ) return `**${ children } **` ;
27+ if ( node . type === "delete" ) return `~~${ children } ~~` ;
28+ return children ;
29+ } ;
30+ const link = ( url : string ) => `[${ url } ](${ / ^ w w w \. / i. test ( url ) ? `http://${ url } ` : url } )` ;
31+ for ( const url of [
32+ "https://x.com/dimasike_" ,
33+ "http://example.com/name__" ,
34+ "HTTPS://example.com/name_" ,
35+ "https://example.com/a_(b)_" ,
36+ "https://example.com/?name=value_" ,
37+ "https://example.com/#section_" ,
38+ "https://example.com/a_?name=value_#section_" ,
39+ "https://example.com/name~" ,
40+ "https://example.com/name~~" ,
41+ "https://example.com/name_~" ,
42+ "https://example.com/name~_" ,
43+ "https://example.com/name__~~_" ,
44+ "https://example.com/?name=value~" ,
45+ "https://example.com/#section~" ,
46+ "www.example.com/name_" ,
47+ "www.example.com/name~_" ,
48+ "WWW.example.com/name~~" ,
49+ "https://example.com/name-" ,
50+ "https://example.com/name/" ,
51+ "https://example.com/name%5F" ,
52+ "https://example.com/name%7E" ,
53+ ] ) {
54+ for ( const punctuation of [ "" , "." , "," , "!" , "?" , ":" , ";" , ")" , "]" , '"' ] ) {
55+ const markdown = `See ${ url } ${ punctuation } next` ;
56+ expect ( shape ( processor . runSync ( processor . parse ( markdown ) , markdown ) as Root ) , markdown ) . toBe (
57+ `See ${ link ( url ) } ${ punctuation } next` ,
58+ ) ;
59+ }
60+ }
61+ for ( const [ markdown , expected ] of [
62+ [ "*https://x.com/dimasike_*" , `*${ link ( "https://x.com/dimasike_" ) } *` ] ,
63+ [ "**https://x.com/dimasike_**" , `**${ link ( "https://x.com/dimasike_" ) } **` ] ,
64+ [ "_https://x.com/dimasike_" , `*${ link ( "https://x.com/dimasike" ) } *` ] ,
65+ [ "__https://x.com/dimasike__" , `**${ link ( "https://x.com/dimasike" ) } **` ] ,
66+ [ "~~https://example.com/name_~~" , `~~${ link ( "https://example.com/name_" ) } ~~` ] ,
67+ [ "~~https://example.com/name~~" , `~~${ link ( "https://example.com/name" ) } ~~` ] ,
68+ [ "~https://example.com/name~" , `~~${ link ( "https://example.com/name" ) } ~~` ] ,
69+ [ "*https://example.com/name~*" , `*${ link ( "https://example.com/name~" ) } *` ] ,
70+ [ "_https://example.com/name~_" , `*${ link ( "https://example.com/name~" ) } *` ] ,
71+ [ "[test](https://example.com/name~)" , "[test](https://example.com/name~)" ] ,
72+ [ "[https://x.com/a](https://x.com/a)~_" , `${ link ( "https://x.com/a" ) } ~_` ] ,
73+ [ "<https://x.com/a>~_" , `${ link ( "https://x.com/a" ) } ~_` ] ,
74+ [ "[test][ref]~_\n\n[ref]: https://x.com/a~_" , "[test][ref]~_" ] ,
75+ [ "`https://example.com/name~_`" , "`https://example.com/name~_`" ] ,
76+ [ "```\nhttps://example.com/name~_\n```" , "```https://example.com/name~_```" ] ,
77+ [ "https\\://example.com/name~_" , link ( "https://example.com/name~_" ) ] ,
78+ [ "https://example.com/name. _~" , `${ link ( "https://example.com/name" ) } . _~` ] ,
79+ 80+ 81+ [ "[test](https://x.com/dimasike_)" , "[test](https://x.com/dimasike_)" ] ,
82+ [ "[https://x.com/a](https://x.com/a)_" , `${ link ( "https://x.com/a" ) } _` ] ,
83+ [ "<https://x.com/a>_" , `${ link ( "https://x.com/a" ) } _` ] ,
84+ [ "[test][ref]_\n\n[ref]: https://x.com/a_" , "[test][ref]_" ] ,
85+ [ "`https://x.com/dimasike_`" , "`https://x.com/dimasike_`" ] ,
86+ [ "```\nhttps://x.com/dimasike_\n```" , "```https://x.com/dimasike_```" ] ,
87+ [ "https://x.com/a. _" , `${ link ( "https://x.com/a" ) } . _` ] ,
88+ [ "https\\://x.com/dimasike_" , link ( "https://x.com/dimasike_" ) ] ,
89+ [ "https://x.com/a\\_" , link ( "https://x.com/a\\_" ) ] ,
90+ [ "https://x.com/a_" , `${ link ( "https://x.com/a_" ) } ;` ] ,
91+ [
92+ "https://x.com/a_ https://x.com/b_" ,
93+ `${ link ( "https://x.com/a_" ) } ${ link ( "https://x.com/b_" ) } ` ,
94+ ] ,
95+ ] as const ) {
96+ expect ( shape ( processor . runSync ( processor . parse ( markdown ) , markdown ) as Root ) , markdown ) . toBe (
97+ expected ,
98+ ) ;
99+ }
100+ for ( const url of [ "https://x.com/dimasike_" , "www.example.com/name~_" ] ) {
101+ const markdown = `${ url } , next` ;
102+ const boundary = { line : 1 , column : url . length + 1 , offset : url . length } ;
103+ expect ( processor . runSync ( processor . parse ( markdown ) , markdown ) ) . toMatchObject ( {
104+ children : [
105+ {
106+ children : [
107+ {
108+ type : "link" ,
109+ url : / ^ w w w \. / i. test ( url ) ? `http://${ url } ` : url ,
110+ position : { end : boundary } ,
111+ children : [ { type : "text" , value : url , position : { end : boundary } } ] ,
112+ } ,
113+ { type : "text" , value : ", next" , position : { start : boundary } } ,
114+ ] ,
115+ } ,
116+ ] ,
117+ } ) ;
118+ }
119+ } ) ;
7120
8121it ( "preserves in-app thread links through the shared Markdown sanitizer" , ( ) => {
9122 const processor = unified ( )
0 commit comments