Skip to content

Commit ea2d979

Browse files
SamMorrowDrumsSamMorrowDrums
committed
Filter invisible Unicode after HTML entity normalization
FilterInvisibleCharacters previously ran only before FilterHTMLTags, so numeric HTML entities (e.g. ​ or ​) that bluemonday decodes into invisible or bidirectional control characters could survive sanitization untouched. Sanitize now applies the invisible-character filter both before HTML processing (so raw invisible characters don't interfere with code-fence parsing) and again after, so entity-decoded characters cannot escape the policy. Also expands the removal set to include: - ARABIC LETTER MARK (U+061C), a directional format character in the same family as the already-covered LRM/RLM marks. - Variation selectors (U+FE00-U+FE0F) and the variation selectors supplement (U+E0100-U+E01EF), which can be used to hide payloads after emoji or other base characters. Fixes #3101
1 parent 912cce6 commit ea2d979

2 files changed

Lines changed: 133 additions & 2 deletions

File tree

‎pkg/sanitize/sanitize.go‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,24 @@ var policy *bluemonday.Policy
1212
var policyOnce sync.Once
1313

1414
func Sanitize(input string) string {
15-
return FilterHTMLTags(FilterCodeFenceMetadata(FilterInvisibleCharacters(input)))
15+
// FilterInvisibleCharacters runs both before and after HTML processing.
16+
// The first pass strips raw invisible characters so they don't interfere
17+
// with code-fence parsing. HTML sanitization (FilterHTMLTags) decodes
18+
// character entities (e.g. "​" or "​" become U+200B), which
19+
// can introduce invisible or bidirectional characters that were not
20+
// present as literal runes in the original input. The second pass
21+
// filters the fully normalized output so entity-encoded characters
22+
// cannot survive the policy.
23+
return FilterInvisibleCharacters(FilterHTMLTags(FilterCodeFenceMetadata(FilterInvisibleCharacters(input))))
1624
}
1725

1826
// FilterInvisibleCharacters removes invisible or control characters that should not appear
1927
// in user-facing titles or bodies. This includes:
2028
// - Unicode tag characters: U+E0001, U+E0020–U+E007F
2129
// - BiDi control characters: U+202A–U+202E, U+2066–U+2069
22-
// - Hidden modifier characters: U+200B, U+200C, U+200E, U+200F, U+00AD, U+FEFF, U+180E, U+2060–U+2064
30+
// - BiDi/directional marks: U+200E, U+200F, U+061C
31+
// - Hidden modifier characters: U+200B, U+200C, U+00AD, U+FEFF, U+180E, U+2060–U+2064
32+
// - Variation selectors: U+FE00–U+FE0F, U+E0100–U+E01EF
2333
func FilterInvisibleCharacters(input string) string {
2434
if input == "" {
2535
return input
@@ -179,6 +189,7 @@ func shouldRemoveRune(r rune) bool {
179189
0x200C, // ZERO WIDTH NON-JOINER
180190
0x200E, // LEFT-TO-RIGHT MARK
181191
0x200F, // RIGHT-TO-LEFT MARK
192+
0x061C, // ARABIC LETTER MARK
182193
0x00AD, // SOFT HYPHEN
183194
0xFEFF, // ZERO WIDTH NO-BREAK SPACE
184195
0x180E: // MONGOLIAN VOWEL SEPARATOR
@@ -204,6 +215,14 @@ func shouldRemoveRune(r rune) bool {
204215
if r >= 0x2060 && r <= 0x2064 {
205216
return true
206217
}
218+
// Variation selectors: U+FE00–U+FE0F
219+
if r >= 0xFE00 && r <= 0xFE0F {
220+
return true
221+
}
222+
// Variation selectors supplement: U+E0100–U+E01EF
223+
if r >= 0xE0100 && r <= 0xE01EF {
224+
return true
225+
}
207226

208227
return false
209228
}

‎pkg/sanitize/sanitize_test.go‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,26 @@ func TestFilterInvisibleCharacters(t *testing.T) {
112112
input: "This is a\u200B bug report.\n\nSteps to reproduce:\u200C\n1. Do this\u200E\n2. Do that\u200F",
113113
expected: "This is a bug report.\n\nSteps to reproduce:\n1. Do this\n2. Do that",
114114
},
115+
{
116+
name: "text with arabic letter mark",
117+
input: "Hello\u061CWorld",
118+
expected: "HelloWorld",
119+
},
120+
{
121+
name: "text with variation selector",
122+
input: "Hello\uFE0FWorld",
123+
expected: "HelloWorld",
124+
},
125+
{
126+
name: "text with variation selector supplement",
127+
input: "Hello\U000E0100World",
128+
expected: "HelloWorld",
129+
},
130+
{
131+
name: "emoji variation selector hidden after emoji (steganography)",
132+
input: "\U0001F600\uFE0F\U000E0101Hi",
133+
expected: "\U0001F600Hi",
134+
},
115135
}
116136

117137
for _, tt := range tests {
@@ -166,6 +186,23 @@ func TestShouldRemoveRune(t *testing.T) {
166186
{name: "before hidden modifier range", rune: 0x205F, expected: false},
167187
{name: "after hidden modifier range", rune: 0x2065, expected: false},
168188

189+
// Additional directional mark
190+
{name: "arabic letter mark", rune: 0x061C, expected: true},
191+
192+
// Range tests - Variation selectors: U+FE00–U+FE0F
193+
{name: "variation selector range start", rune: 0xFE00, expected: true},
194+
{name: "variation selector range middle", rune: 0xFE05, expected: true},
195+
{name: "variation selector range end (VS16, emoji presentation)", rune: 0xFE0F, expected: true},
196+
{name: "before variation selector range", rune: 0xFDFF, expected: false},
197+
{name: "after variation selector range", rune: 0xFE10, expected: false},
198+
199+
// Range tests - Variation selectors supplement: U+E0100–U+E01EF
200+
{name: "variation selector supplement range start", rune: 0xE0100, expected: true},
201+
{name: "variation selector supplement range middle", rune: 0xE0150, expected: true},
202+
{name: "variation selector supplement range end", rune: 0xE01EF, expected: true},
203+
{name: "before variation selector supplement range", rune: 0xE00FF, expected: false},
204+
{name: "after variation selector supplement range", rune: 0xE01F0, expected: false},
205+
169206
// Characters that should NOT be removed
170207
{name: "regular ascii letter", rune: 'A', expected: false},
171208
{name: "regular ascii digit", rune: '1', expected: false},
@@ -300,3 +337,78 @@ func TestSanitizeRemovesInvisibleCodeFenceMetadata(t *testing.T) {
300337
result := Sanitize(input)
301338
assert.Equal(t, expected, result)
302339
}
340+
341+
// TestSanitizeFiltersInvisibleCharactersAfterEntityDecoding covers the core
342+
// regression from issue #3101: invisible/bidi characters encoded as HTML
343+
// character entities are decoded by FilterHTMLTags, so the invisible-character
344+
// policy must also run after HTML processing, not only on the raw input.
345+
func TestSanitizeFiltersInvisibleCharactersAfterEntityDecoding(t *testing.T) {
346+
tests := []struct {
347+
name string
348+
input string
349+
expected string
350+
}{
351+
{
352+
name: "decimal entity for zero width space",
353+
input: "Hello&#8203;World",
354+
expected: "HelloWorld",
355+
},
356+
{
357+
name: "hexadecimal entity for zero width space",
358+
input: "Hello&#x200B;World",
359+
expected: "HelloWorld",
360+
},
361+
{
362+
name: "hexadecimal entity for zero width space (lowercase x, uppercase hex)",
363+
input: "Hello&#x200b;World",
364+
expected: "HelloWorld",
365+
},
366+
{
367+
name: "decimal entity for right-to-left override",
368+
input: "Hello&#8238;World",
369+
expected: "HelloWorld",
370+
},
371+
{
372+
name: "hexadecimal entity for left-to-right override",
373+
input: "Hello&#x202D;World",
374+
expected: "HelloWorld",
375+
},
376+
{
377+
name: "decimal entity for variation selector",
378+
input: "Hello&#65039;World",
379+
expected: "HelloWorld",
380+
},
381+
{
382+
name: "hexadecimal entity for variation selector supplement",
383+
input: "Hello&#xE0100;World",
384+
expected: "HelloWorld",
385+
},
386+
{
387+
name: "direct invisible rune alongside entity encoded one",
388+
input: "Hello\u200B&#8206;World",
389+
expected: "HelloWorld",
390+
},
391+
{
392+
name: "entity for ordinary ascii character is preserved",
393+
input: "Hello&#65;World",
394+
expected: "HelloAWorld",
395+
},
396+
{
397+
name: "entity for benign unicode character is preserved",
398+
input: "Hello&#19990;World", // &#19990; is 世
399+
expected: "Hello世World",
400+
},
401+
{
402+
name: "benign unicode text without entities is untouched",
403+
input: "Hello 世界 🌍 αβγ",
404+
expected: "Hello 世界 🌍 αβγ",
405+
},
406+
}
407+
408+
for _, tt := range tests {
409+
t.Run(tt.name, func(t *testing.T) {
410+
result := Sanitize(tt.input)
411+
assert.Equal(t, tt.expected, result)
412+
})
413+
}
414+
}

0 commit comments

Comments
 (0)