|
32 | 32 | # ASCII substitutes. Ordinary CJK punctuation (,。) and ideographs must not |
33 | 33 | # be flagged. |
34 | 34 | _CONFUSABLE_RE = re.compile(r"[\uff10-\uff19\uff21-\uff3a\uff41-\uff5a\u24b6-\u24cf\u24d0-\u24e9]+") |
35 | | -# Mathematical alphanumeric symbols (U+1D400-U+1D7FF: bold, italic, script, |
36 | | -# fraktur, double-struck, sans-serif, monospace letters/digits) normalize |
37 | | -# straight to ASCII under NFKC, same as the confusables above. Unlike those, |
38 | | -# ordinary math notation strings several styled variables together ("let |
39 | | -# f(x) = ax + b", "define xy as the product"), so this block can't be |
40 | | -# unconditionally suspicious at any short run length. Benign notation tops |
41 | | -# out at 2 adjacent styled characters; a payload rendered in math-bold runs |
42 | | -# 9-12. The threshold sits between the two: below it is notation, at or |
43 | | -# above it is a styled word. |
| 35 | +# Mathematical alphanumeric letters (U+1D400-U+1D6A3: bold, italic, script, |
| 36 | +# fraktur, double-struck, sans-serif, monospace) normalize straight to ASCII |
| 37 | +# under NFKC, same as the confusables above. The Greek and digit subranges |
| 38 | +# after U+1D6A3 do not normalize to ASCII letters, so they cannot spell an |
| 39 | +# instruction and are left out. Ordinary math notation strings a few styled |
| 40 | +# variables together ("let f(x) = ax + b"), so a run only counts as evasion |
| 41 | +# at _MATH_ALPHANUMERIC_MIN_RUN characters: benign notation tops out at 2-3 |
| 42 | +# adjacent styled characters, while a payload in math-bold runs 9-12. |
44 | 43 | _MATH_ALPHANUMERIC_MIN_RUN = 4 |
45 | | -_MATH_ALPHANUMERIC_RE = re.compile(r"[\U0001d400-\U0001d7ff]+") |
| 44 | +_MATH_ALPHANUMERIC_RE = re.compile(r"[\U0001d400-\U0001d6a3]+") |
46 | 45 | _WORD_RE = re.compile(r"\w+") |
47 | 46 |
|
48 | 47 | # Only collapse evasion spans that sit close together (e.g. zero-width chars |
@@ -85,10 +84,14 @@ def _group_whitespace_runs( |
85 | 84 | the grouped runs' own lengths, not end - start: measuring span width |
86 | 85 | would let a wide whitespace gap between two short runs *dilute* the |
87 | 86 | count exactly the way this exists to prevent, in the other direction. |
| 87 | +
|
| 88 | + Single-character runs are not grouped: spaced single variables ("let a b |
| 89 | + c d be reals") are ordinary notation, and a chunked payload's fragments |
| 90 | + are longer than one character. |
88 | 91 | """ |
| 92 | + runs = sorted(run for run in runs if run[1] - run[0] > 1) |
89 | 93 | if not runs: |
90 | 94 | return [] |
91 | | - runs = sorted(runs) |
92 | 95 | groups: list[list[tuple[int, int]]] = [[runs[0]]] |
93 | 96 | for span_start, span_end in runs[1:]: |
94 | 97 | _, last_end = groups[-1][-1] |
|
0 commit comments