Skip to content

Commit ab5ade8

Browse files
spicysharkcorthalley
authored andcommitted
end a line at a carriage return in the tokenizer (#1387)
The carriage return of CRLF line ends stayed in the last token of each line, so a name there was read with a stray \013 label and a zone with CRLF line ends was rejected. Read a carriage return outside a quoted string as a line end, and skip the line feed that follows it. (cherry picked from commit a1bbca8)
1 parent d9d259d commit ab5ade8

4 files changed

Lines changed: 60 additions & 0 deletions

File tree

‎dns/tokenizer.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,9 @@ class Tokenizer:
224224
225225
eof: This variable is true if the tokenizer has encountered EOF.
226226
227+
after_cr: This variable is true if the most recent character read was
228+
a carriage return, which is read as a newline.
229+
227230
delimiters: The current delimiter dictionary.
228231
229232
line_number: The current line number
@@ -275,6 +278,7 @@ def __init__(
275278
self.multiline = 0
276279
self.quoting = False
277280
self.eof = False
281+
self.after_cr = False
278282
self.delimiters = _DELIMITERS
279283
self.line_number = 1
280284
assert filename is not None
@@ -292,6 +296,15 @@ def _get_char(self) -> str:
292296
c = ""
293297
else:
294298
c = self.file.read(1)
299+
if self.after_cr:
300+
self.after_cr = False
301+
if c == "\n":
302+
# The newline of a CRLF pair; the CR already ended the line.
303+
c = self.file.read(1)
304+
if c == "\r" and not self.quoting:
305+
# Outside of a quoted string, a CR ends the line like a newline.
306+
self.after_cr = True
307+
c = "\n"
295308
if c == "":
296309
self.eof = True
297310
elif c == "\n":

‎doc/whatsnew.rst‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@ What's New in dnspython
66
2.9.1 (in development)
77
----------------------
88

9+
* ``dns.tokenizer.Tokenizer`` now ends a line at a carriage return, or at a
10+
carriage return followed by a line feed, unless the carriage return is in a
11+
quoted string. The carriage return of CRLF line ends used to stay in the
12+
last token of each line and in the text of a comment, so e.g.
13+
``dns.zone.from_text()`` rejected a zone with CRLF line ends.
14+
915
* DNSSEC validation no longer uses a DNSKEY with the REVOKE flag set to
1016
validate anything other than the DNSKEY RRset. RFC 5011 section 2.1 says a
1117
revoked key must not be used for any other purpose, but it was a candidate

‎tests/test_tokenizer.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,35 @@ def testEOL(self):
8181
token2 = tok.get()
8282
self.assertTrue(token1.is_eol() and token2.is_eof())
8383

84+
def testCRLF(self):
85+
def read_all(text):
86+
tok = dns.tokenizer.Tokenizer(text)
87+
tokens = []
88+
while True:
89+
token = tok.get(want_leading=True, want_comment=True)
90+
tokens.append((token.ttype, token.value))
91+
if token.is_eof():
92+
return tokens, tok.line_number
93+
94+
# CRLF reads as LF does, in the cases where the CR could be kept:
95+
# at the end of a token, of a comment, and of a blank line.
96+
text = "a b ; c\nd (\n e ; f\n g)\n\n \nh\n"
97+
expected = read_all(text)
98+
self.assertEqual(read_all(text.replace("\n", "\r\n")), expected)
99+
self.assertEqual(read_all(text.replace("\n", "\r\n").encode()), expected)
100+
# A CR alone also ends the line.
101+
self.assertEqual(read_all(text.replace("\n", "\r")), expected)
102+
# A CR alone does not skip a later LF.
103+
self.assertEqual(read_all("a\rb\n"), read_all("a\nb\n"))
104+
# A CR in a quoted string is data.
105+
tok = dns.tokenizer.Tokenizer('"a\rb"\r\n')
106+
self.assertEqual(tok.get(), Token(dns.tokenizer.QUOTED_STRING, "a\rb"))
107+
self.assertTrue(tok.get().is_eol())
108+
self.assertTrue(tok.get().is_eof())
109+
tok = dns.tokenizer.Tokenizer('"a\r\nb"')
110+
with self.assertRaises(dns.exception.SyntaxError):
111+
tok.get()
112+
84113
def testWS1(self):
85114
tok = dns.tokenizer.Tokenizer(" \n")
86115
token1 = tok.get()

‎tests/test_zone.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -594,6 +594,18 @@ def testFromText(self):
594594
f.write("\n")
595595
self.assertEqual(f.getvalue(), example_text_output)
596596

597+
def testFromTextCRLF(self):
598+
z = dns.zone.from_text(
599+
example_text.replace("\n", "\r\n"), "example.", relativize=True
600+
)
601+
self.assertEqual(z, dns.zone.from_text(example_text, "example.", True))
602+
z = dns.zone.from_text(
603+
example_comments_text.replace("\n", "\r\n"), "example.", relativize=True
604+
)
605+
f = StringIO()
606+
z.to_file(f, want_comments=True)
607+
self.assertEqual(f.getvalue(), example_comments_text_output)
608+
597609
def testGenerate(self):
598610
z = dns.zone.from_text(example_generate, "example.", relativize=True)
599611
f = StringIO()

0 commit comments

Comments
 (0)