Skip to content

Commit aed9336

Browse files
authored
fix: case fold reference link labels (#4077)
Labels were matched with toLowerCase(), which is not the Unicode case fold CommonMark asks for, so labels that only agree once folded never matched: [ẞ] [Straße] [SS]: /url [STRASSE]: /url 'ẞ'.toLowerCase() is 'ß', never 'ss'. Round-tripping through upper case reaches the fold, the way commonmark.js normalizeReference() does; lowercasing afterwards keeps def.tag and the tokens.links keys in the lower case form they have always had. The two link-map lookups in Lexer used the label verbatim, so they went through the same helper: without it a folded key would stop the reflink masking from finding definitions it used to find. This turns on CommonMark example 540 (Links 81/90 -> 82/90).
1 parent 123ce04 commit aed9336

7 files changed

Lines changed: 26 additions & 8 deletions

File tree

‎src/Lexer.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { _Tokenizer } from './Tokenizer.ts';
22
import { _defaults } from './defaults.ts';
33
import { other, block, inline } from './rules.ts';
4+
import { normalizeLabel } from './helpers.ts';
45
import type { Token, TokensList, Tokens } from './Tokens.ts';
56
import type { MarkedOptions } from './MarkedOptions.ts';
67

@@ -321,7 +322,7 @@ export class _Lexer<ParserOutput = string, RendererOutput = string> {
321322
for (const match of text.matchAll(this.tokenizer.rules.inline.reflinkSearch)) {
322323
const match0 = match[0];
323324
const refStart = match0.lastIndexOf('[');
324-
if (match0.charAt(0) === '!' || !Object.hasOwn(this.tokens.links, match0.slice(refStart + 1, -1))) {
325+
if (match0.charAt(0) === '!' || !Object.hasOwn(this.tokens.links, normalizeLabel(match0.slice(refStart + 1, -1)))) {
325326
continue;
326327
}
327328
// a candidate holding a link is not a link either, so it does not count
@@ -347,7 +348,7 @@ export class _Lexer<ParserOutput = string, RendererOutput = string> {
347348
const reflinkSearch = this.tokenizer.rules.inline.reflinkSearch;
348349
const maskReflink = (match0: string): string => {
349350
const refStart = match0.lastIndexOf('[');
350-
if (!Object.hasOwn(this.tokens.links, match0.slice(refStart + 1, -1))) {
351+
if (!Object.hasOwn(this.tokens.links, normalizeLabel(match0.slice(refStart + 1, -1)))) {
351352
return match0;
352353
}
353354
// CommonMark: "Links may not contain other links, at any level of

‎src/Tokenizer.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import {
44
splitCells,
55
findClosingBracket,
66
expandTabs,
7+
normalizeLabel,
78
trimTrailingBlankLines,
89
} from './helpers.ts';
910
import type { Rules } from './rules.ts';
@@ -527,7 +528,7 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
527528
def(src: string): Tokens.Def | undefined {
528529
const cap = this.rules.block.def.exec(src);
529530
if (cap) {
530-
const tag = cap[1].toLowerCase().replace(this.rules.other.multipleSpaceGlobal, ' ');
531+
const tag = normalizeLabel(cap[1]).replace(this.rules.other.multipleSpaceGlobal, ' ');
531532
const href = cap[2] ? cap[2].replace(this.rules.other.hrefBrackets, '$1').replace(this.rules.inline.anyPunctuation, '$1') : '';
532533
const title = cap[3] ? cap[3].substring(1, cap[3].length - 1).replace(this.rules.inline.anyPunctuation, '$1') : cap[3];
533534
return {
@@ -746,7 +747,7 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
746747
if ((cap = this.rules.inline.reflink.exec(src))
747748
|| (cap = this.rules.inline.nolink.exec(src))) {
748749
const linkString = (cap[2] || cap[1]).replace(this.rules.other.multipleSpaceGlobal, ' ');
749-
const link = links[linkString.toLowerCase()];
750+
const link = links[normalizeLabel(linkString)];
750751
if (!link) {
751752
const text = cap[0].charAt(0);
752753
return {

‎src/helpers.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,17 @@ export function trimTrailingBlankLines(str: string) {
123123
return lines.slice(0, end + 1).join('\n');
124124
}
125125

126+
/**
127+
* Normalizes a link label so definitions and references can be matched.
128+
* CommonMark asks for a Unicode case fold, which `toLowerCase()` does not
129+
* reach: `ẞ` lowercases to `ß` and so never meets `SS`. Round-tripping
130+
* through upper case does, as in commonmark.js; the final `toLowerCase()`
131+
* keeps the folded label lower case, the form `def.tag` has always used.
132+
*/
133+
export function normalizeLabel(label: string) {
134+
return label.toLowerCase().toUpperCase().toLowerCase();
135+
}
136+
126137
export function findClosingBracket(str: string, b: string) {
127138
if (str.indexOf(b[1]) === -1) {
128139
return -1;

‎test/specs/commonmark/commonmark.0.31.2.json‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4324,8 +4324,7 @@
43244324
"example": 540,
43254325
"start_line": 8129,
43264326
"end_line": 8135,
4327-
"section": "Links",
4328-
"shouldFail": true
4327+
"section": "Links"
43294328
},
43304329
{
43314330
"markdown": "[Foo\n bar]: /url\n\n[Baz][Foo bar]\n",

‎test/specs/gfm/commonmark.0.31.2.json‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4324,8 +4324,7 @@
43244324
"example": 540,
43254325
"start_line": 8129,
43264326
"end_line": 8135,
4327-
"section": "Links",
4328-
"shouldFail": true
4327+
"section": "Links"
43294328
},
43304329
{
43314330
"markdown": "[Foo\n bar]: /url\n\n[Baz][Foo bar]\n",
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
<p><a href="/sharp-s">ẞ</a> and <a href="/street">Straße</a> and <a href="/digraph">Dž</a> and <a href="/ligature">fi</a>.</p>
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[ẞ] and [Straße] and [Dž] and [fi].
2+
3+
[SS]: /sharp-s
4+
[STRASSE]: /street
5+
[DŽ]: /digraph
6+
[FI]: /ligature

0 commit comments

Comments
 (0)