Skip to content

Commit c555f88

Browse files
serhiy-storchakaezio-melottiwaylan
authored
[3.12] gh-135661: Fix parsing start and end tags in HTMLParser according to the HTML5 standard (GH-135930) (GH-136268)
* Whitespaces no longer accepted between `</` and the tag name. E.g. `</ script>` does not end the script section. * Vertical tabulation (`\v`) and non-ASCII whitespaces no longer recognized as whitespaces. The only whitespaces are `\t\n\r\f `. * Null character (U+0000) no longer ends the tag name. * Attributes and slashes after the tag name in end tags are now ignored, instead of terminating after the first `>` in quoted attribute value. E.g. `</script/foo=">"/>`. * Multiple slashes and whitespaces between the last attribute and closing `>` are now ignored in both start and end tags. E.g. `<a foo=bar/ //>`. * Multiple `=` between attribute name and value are no longer collapsed. E.g. `<a foo==bar>` produces attribute "foo" with value "=bar". * Whitespaces between the `=` separator and attribute name or value are no longer ignored. E.g. `<a foo =bar>` produces two attributes "foo" and "=bar", both with value None; `<a foo= bar>` produces two attributes: "foo" with value "" and "bar" with value None. * Fix data loss after unclosed script or style tag (gh-86155). Also backport test.support.subTests() (gh-135120). --------- (cherry picked from commit 0243f97) Co-authored-by: Ezio Melotti <[email protected]> Co-authored-by: Waylan Limberg <[email protected]>
1 parent ab0893f commit c555f88

File tree

5 files changed

+222
-120
lines changed

5 files changed

+222
-120
lines changed

‎Lib/html/parser.py‎

Lines changed: 70 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,43 @@
2929
piclose = re.compile('>')
3030
commentclose = re.compile(r'--\s*>')
3131
# Note:
32-
# 1) if you change tagfind/attrfind remember to update locatestarttagend too;
33-
# 2) if you change tagfind/attrfind and/or locatestarttagend the parser will
32+
# 1) if you change tagfind/attrfind remember to update locatetagend too;
33+
# 2) if you change tagfind/attrfind and/or locatetagend the parser will
3434
# explode, so don't do it.
35-
# see http://www.w3.org/TR/html5/tokenization.html#tag-open-state
36-
# and http://www.w3.org/TR/html5/tokenization.html#tag-name-state
37-
tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />\x00]*)(?:\s|/(?!>))*')
38-
attrfind_tolerant = re.compile(
39-
r'((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*'
40-
r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*')
35+
# see the HTML5 specs section "13.2.5.6 Tag open state",
36+
# "13.2.5.8 Tag name state" and "13.2.5.33 Attribute name state".
37+
# https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state
38+
# https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
39+
# https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state
40+
tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />]*)(?:[\t\n\r\f ]|/(?!>))*')
41+
attrfind_tolerant = re.compile(r"""
42+
(
43+
(?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name
44+
)
45+
(= # value indicator
46+
('[^']*' # LITA-enclosed value
47+
|"[^"]*" # LIT-enclosed value
48+
|(?!['"])[^>\t\n\r\f ]* # bare value
49+
)
50+
)?
51+
(?:[\t\n\r\f ]|/(?!>))* # possibly followed by a space
52+
""", re.VERBOSE)
53+
locatetagend = re.compile(r"""
54+
[a-zA-Z][^\t\n\r\f />]* # tag name
55+
[\t\n\r\f /]* # optional whitespace before attribute name
56+
(?:(?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name
57+
(?:= # value indicator
58+
(?:'[^']*' # LITA-enclosed value
59+
|"[^"]*" # LIT-enclosed value
60+
|(?!['"])[^>\t\n\r\f ]* # bare value
61+
)
62+
)?
63+
[\t\n\r\f /]* # possibly followed by a space
64+
)*
65+
>?
66+
""", re.VERBOSE)
67+
# The following variables are not used, but are temporarily left for
68+
# backward compatibility.
4169
locatestarttagend_tolerant = re.compile(r"""
4270
<[a-zA-Z][^\t\n\r\f />\x00]* # tag name
4371
(?:[\s/]* # optional whitespace before attribute name
@@ -54,8 +82,6 @@
5482
\s* # trailing whitespace
5583
""", re.VERBOSE)
5684
endendtag = re.compile('>')
57-
# the HTML 5 spec, section 8.1.2.2, doesn't allow spaces between
58-
# </ and the tag name, so maybe this should be fixed
5985
endtagfind = re.compile(r'</\s*([a-zA-Z][-.a-zA-Z0-9:_]*)\s*>')
6086

6187

@@ -123,7 +149,8 @@ def get_starttag_text(self):
123149

124150
def set_cdata_mode(self, elem):
125151
self.cdata_elem = elem.lower()
126-
self.interesting = re.compile(r'</\s*%s\s*>' % self.cdata_elem, re.I)
152+
self.interesting = re.compile(r'</%s(?=[\t\n\r\f />])' % self.cdata_elem,
153+
re.IGNORECASE|re.ASCII)
127154

128155
def clear_cdata_mode(self):
129156
self.interesting = interesting_normal
@@ -148,7 +175,7 @@ def goahead(self, end):
148175
# & near the end and see if it's followed by a space or ;.
149176
amppos = rawdata.rfind('&', max(i, n-34))
150177
if (amppos >= 0 and
151-
not re.compile(r'[\s;]').search(rawdata, amppos)):
178+
not re.compile(r'[\t\n\r\f ;]').search(rawdata, amppos)):
152179
break # wait till we get all the text
153180
j = n
154181
else:
@@ -261,7 +288,7 @@ def goahead(self, end):
261288
else:
262289
assert 0, "interesting.search() lied"
263290
# end while
264-
if end and i < n and not self.cdata_elem:
291+
if end and i < n:
265292
if self.convert_charrefs and not self.cdata_elem:
266293
self.handle_data(unescape(rawdata[i:n]))
267294
else:
@@ -292,7 +319,7 @@ def parse_html_declaration(self, i):
292319
return self.parse_bogus_comment(i)
293320

294321
# Internal -- parse bogus comment, return length or -1 if not terminated
295-
# see http://www.w3.org/TR/html5/tokenization.html#bogus-comment-state
322+
# see https://html.spec.whatwg.org/multipage/parsing.html#bogus-comment-state
296323
def parse_bogus_comment(self, i, report=1):
297324
rawdata = self.rawdata
298325
assert rawdata[i:i+2] in ('<!', '</'), ('unexpected call to '
@@ -318,6 +345,8 @@ def parse_pi(self, i):
318345

319346
# Internal -- handle starttag, return end or -1 if not terminated
320347
def parse_starttag(self, i):
348+
# See the HTML5 specs section "13.2.5.8 Tag name state"
349+
# https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
321350
self.__starttag_text = None
322351
endpos = self.check_for_whole_start_tag(i)
323352
if endpos < 0:
@@ -363,76 +392,42 @@ def parse_starttag(self, i):
363392
# or -1 if incomplete.
364393
def check_for_whole_start_tag(self, i):
365394
rawdata = self.rawdata
366-
m = locatestarttagend_tolerant.match(rawdata, i)
367-
if m:
368-
j = m.end()
369-
next = rawdata[j:j+1]
370-
if next == ">":
371-
return j + 1
372-
if next == "/":
373-
if rawdata.startswith("/>", j):
374-
return j + 2
375-
if rawdata.startswith("/", j):
376-
# buffer boundary
377-
return -1
378-
# else bogus input
379-
if j > i:
380-
return j
381-
else:
382-
return i + 1
383-
if next == "":
384-
# end of input
385-
return -1
386-
if next in ("abcdefghijklmnopqrstuvwxyz=/"
387-
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
388-
# end of input in or before attribute value, or we have the
389-
# '/' from a '/>' ending
390-
return -1
391-
if j > i:
392-
return j
393-
else:
394-
return i + 1
395-
raise AssertionError("we should not get here!")
395+
match = locatetagend.match(rawdata, i+1)
396+
assert match
397+
j = match.end()
398+
if rawdata[j-1] != ">":
399+
return -1
400+
return j
396401

397402
# Internal -- parse endtag, return end or -1 if incomplete
398403
def parse_endtag(self, i):
404+
# See the HTML5 specs section "13.2.5.7 End tag open state"
405+
# https://html.spec.whatwg.org/multipage/parsing.html#end-tag-open-state
399406
rawdata = self.rawdata
400407
assert rawdata[i:i+2] == "</", "unexpected call to parse_endtag"
401-
match = endendtag.search(rawdata, i+1) # >
402-
if not match:
408+
if rawdata.find('>', i+2) < 0: # fast check
403409
return -1
404-
gtpos = match.end()
405-
match = endtagfind.match(rawdata, i) # </ + tag + >
406-
if not match:
407-
if self.cdata_elem is not None:
408-
self.handle_data(rawdata[i:gtpos])
409-
return gtpos
410-
# find the name: w3.org/TR/html5/tokenization.html#tag-name-state
411-
namematch = tagfind_tolerant.match(rawdata, i+2)
412-
if not namematch:
413-
# w3.org/TR/html5/tokenization.html#end-tag-open-state
414-
if rawdata[i:i+3] == '</>':
415-
return i+3
416-
else:
417-
return self.parse_bogus_comment(i)
418-
tagname = namematch.group(1).lower()
419-
# consume and ignore other stuff between the name and the >
420-
# Note: this is not 100% correct, since we might have things like
421-
# </tag attr=">">, but looking for > after the name should cover
422-
# most of the cases and is much simpler
423-
gtpos = rawdata.find('>', namematch.end())
424-
self.handle_endtag(tagname)
425-
return gtpos+1
410+
if not endtagopen.match(rawdata, i): # </ + letter
411+
if rawdata[i+2:i+3] == '>': # </> is ignored
412+
# "missing-end-tag-name" parser error
413+
return i+3
414+
else:
415+
return self.parse_bogus_comment(i)
426416

427-
elem = match.group(1).lower() # script or style
428-
if self.cdata_elem is not None:
429-
if elem != self.cdata_elem:
430-
self.handle_data(rawdata[i:gtpos])
431-
return gtpos
417+
match = locatetagend.match(rawdata, i+2)
418+
assert match
419+
j = match.end()
420+
if rawdata[j-1] != ">":
421+
return -1
432422

433-
self.handle_endtag(elem)
423+
# find the name: "13.2.5.8 Tag name state"
424+
# https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
425+
match = tagfind_tolerant.match(rawdata, i+2)
426+
assert match
427+
tag = match.group(1).lower()
428+
self.handle_endtag(tag)
434429
self.clear_cdata_mode()
435-
return gtpos
430+
return j
436431

437432
# Overridable -- finish processing of start+end tag: <tag.../>
438433
def handle_startendtag(self, tag, attrs):

‎Lib/test/support/__init__.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -867,6 +867,31 @@ def check_sizeof(test, o, size):
867867
% (type(o), result, size)
868868
test.assertEqual(result, size, msg)
869869

870+
def subTests(arg_names, arg_values, /, *, _do_cleanups=False):
871+
"""Run multiple subtests with different parameters.
872+
"""
873+
single_param = False
874+
if isinstance(arg_names, str):
875+
arg_names = arg_names.replace(',',' ').split()
876+
if len(arg_names) == 1:
877+
single_param = True
878+
arg_values = tuple(arg_values)
879+
def decorator(func):
880+
if isinstance(func, type):
881+
raise TypeError('subTests() can only decorate methods, not classes')
882+
@functools.wraps(func)
883+
def wrapper(self, /, *args, **kwargs):
884+
for values in arg_values:
885+
if single_param:
886+
values = (values,)
887+
subtest_kwargs = dict(zip(arg_names, values))
888+
with self.subTest(**subtest_kwargs):
889+
func(self, *args, **kwargs, **subtest_kwargs)
890+
if _do_cleanups:
891+
self.doCleanups()
892+
return wrapper
893+
return decorator
894+
870895
#=======================================================================
871896
# Decorator/context manager for running a code in a different locale,
872897
# correctly resetting it afterwards.

0 commit comments

Comments
 (0)