Skip to content

Commit cead8de

Browse files
authored
fix: decode numeric character references in text (#4076)
1 parent e136da7 commit cead8de

5 files changed

Lines changed: 37 additions & 1 deletion

File tree

‎src/Tokenizer.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import {
55
findClosingBracket,
66
expandTabs,
77
normalizeLabel,
8+
decodeNumericCharacterReferences,
89
trimTrailingBlankLines,
910
} from './helpers.ts';
1011
import type { Rules } from './rules.ts';
@@ -1046,7 +1047,9 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
10461047
return {
10471048
type: 'text',
10481049
raw: cap[0],
1049-
text: cap[0],
1050+
// raw HTML keeps whatever it was written with, everywhere else a numeric
1051+
// character reference stands for the character itself
1052+
text: escaped ? cap[0] : decodeNumericCharacterReferences(cap[0]),
10501053
escaped,
10511054
};
10521055
}

‎src/helpers.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,21 @@ export function escapeHtmlEntities(html: string, encode?: boolean) {
2626
return html;
2727
}
2828

29+
/**
30+
* Numeric character references are recognized outside code and are equivalent to the
31+
* character they name. Values that are zero, out of range, or a surrogate become the
32+
* replacement character.
33+
*/
34+
export function decodeNumericCharacterReferences(text: string) {
35+
return text.replace(other.numericCharacterReference, (_, dec: string, hex: string) => {
36+
const code = dec === undefined ? Number.parseInt(hex, 16) : Number.parseInt(dec, 10);
37+
if (code === 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) {
38+
return '�';
39+
}
40+
return String.fromCodePoint(code);
41+
});
42+
}
43+
2944
export function cleanUrl(href: string) {
3045
try {
3146
href = encodeURI(href).replace(other.percentDecode, '%');

‎src/rules.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ export const other = {
7979
endAngleBracket: />$/,
8080
pedanticHrefTitle: /^([^'"]*[^\s])\s+(['"])(.*)\2/,
8181
unicodeAlphaNumeric: /[\p{L}\p{N}]/u,
82+
numericCharacterReference: /&#(?:(\d{1,7})|[Xx]([A-Fa-f0-9]{1,6}));/g,
8283
escapeTest: /[&<>"']/,
8384
escapeReplace: /[&<>"']/g,
8485
escapeTestNoEncode: /[<>"']|&(?!(#\d{1,7}|#[Xx][a-fA-F0-9]{1,6}|\w+);)/,
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
<p># Ӓ</p>
2+
<p>&quot; ಫ</p>
3+
<p>�</p>
4+
<p>&amp;#abcdef0;</p>
5+
<p><code>&amp;#35;</code></p>
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
---
2+
renderExact: true
3+
---
4+
&#35; &#1234;
5+
6+
&#X22; &#xcab;
7+
8+
&#0;
9+
10+
&#abcdef0;
11+
12+
`&#35;`

0 commit comments

Comments
 (0)