Skip to content

Commit cd770b0

Browse files
pquentinillia-v
andauthored
Merge commit from fork
* Bound chunk-size line reads to prevent memory exhaustion DoS The streaming chunked path (HTTPResponse.stream() / read_chunked()) read each chunk-size line with an unbounded readline(), unlike the plain read() path which delegates to http.client's _MAXLINE-capped reader. A malicious server could advertise Transfer-Encoding: chunked and then send a very long run of bytes with no newline, forcing the client to buffer the entire run before the chunk size could be rejected. This caps the chunk-size line (and the trailing CRLF line) at _MAXLINE, raising ProtocolError when exceeded, matching the behaviour read() already had. Only the size line is bounded; chunk bodies still go through the sized _safe_read(), so legitimate large chunks are unaffected. Verified with a local malicious server streaming a 200 MiB unterminated size line: stream() previously grew RSS by ~530 MB, now stays at ~1 MB and fails fast, matching read(). A 5 MiB chunk body still streams correctly. * Fix a black formatting issue * Test chunked response trailer length limits * Fix number in the changelog entry --------- Co-authored-by: Illia Volochii <[email protected]>
1 parent ea2ad7b commit cd770b0

4 files changed

Lines changed: 132 additions & 3 deletions

File tree

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fixed a denial-of-service issue where a malicious server could cause unbounded memory usage in the streaming chunked-transfer-encoding read path (``HTTPResponse.stream()`` / ``HTTPResponse.read_chunked()``) by sending an unterminated chunk-size line. The chunk-size line and trailing CRLF are now read with a bounded length matching :data:`http.client._MAXLINE`, raising :class:`~urllib3.exceptions.ProtocolError` if exceeded, consistent with the non-streaming ``HTTPResponse.read()`` path.

‎src/urllib3/response.py‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,13 @@
5353
# Read in 64 KiB chunks
5454
_READ_CHUNK_SIZE = 2**16
5555

56+
# Maximum length of a chunk-size line (and the trailing CRLF line) when reading a
57+
# chunked-transfer-encoded response. Matches ``http.client._MAXLINE`` so urllib3's
58+
# streaming path bounds these reads exactly like the stdlib ``.read()`` path does,
59+
# preventing a malicious server from forcing unbounded buffering (memory exhaustion)
60+
# via an unterminated chunk-size line.
61+
_MAX_CHUNK_LINE_LENGTH = 2**16
62+
5663

5764
class ContentDecoder:
5865
def decompress(self, data: bytes, max_length: int = -1) -> bytes:
@@ -1357,7 +1364,12 @@ def _update_chunk_length(self) -> None:
13571364
# we'll try to read it from socket.
13581365
if self.chunk_left is not None:
13591366
return None
1360-
line = self._fp.fp.readline() # type: ignore[union-attr]
1367+
line = self._fp.fp.readline(_MAX_CHUNK_LINE_LENGTH + 1) # type: ignore[union-attr]
1368+
if len(line) > _MAX_CHUNK_LINE_LENGTH:
1369+
self.close()
1370+
raise ProtocolError(
1371+
"Response chunk size line exceeded maximum allowed length"
1372+
) from None
13611373
line = line.split(b";", 1)[0]
13621374
try:
13631375
self.chunk_left = int(line, 16)
@@ -1468,7 +1480,11 @@ def read_chunked(
14681480

14691481
# Chunk content ends with \r\n: discard it.
14701482
while self._fp is not None:
1471-
line = self._fp.fp.readline()
1483+
line = self._fp.fp.readline(_MAX_CHUNK_LINE_LENGTH + 1)
1484+
if len(line) > _MAX_CHUNK_LINE_LENGTH:
1485+
raise ProtocolError(
1486+
"Response chunk trailer line exceeded maximum allowed length"
1487+
)
14721488
if not line:
14731489
# Some sites may not end with '\r\n'.
14741490
break

‎test/test_response.py‎

Lines changed: 79 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
SSLError,
2828
)
2929
from urllib3.response import ( # type: ignore[attr-defined]
30+
_MAX_CHUNK_LINE_LENGTH,
3031
BaseHTTPResponse,
3132
BytesQueueBuffer,
3233
HTTPResponse,
@@ -1890,6 +1891,63 @@ def test_invalid_chunk_length(self) -> None:
18901891
assert isinstance(orig_ex, InvalidChunkLength)
18911892
assert orig_ex.length == fp.BAD_LENGTH_LINE.encode()
18921893

1894+
def test_chunk_size_line_too_long(self) -> None:
1895+
# A malicious server can send a chunk-size line with no newline. urllib3's
1896+
# streaming path must reject it with a bounded read rather than buffering
1897+
# the whole line (memory exhaustion). See GHSA / CVE for chunked DoS.
1898+
stream = [b"foooo"]
1899+
fp = MockChunkedEncodingLongChunkSizeLine(stream)
1900+
r = httplib.HTTPResponse(MockSock) # type: ignore[arg-type]
1901+
r.fp = fp # type: ignore[assignment]
1902+
r.chunked = True
1903+
r.chunk_left = None
1904+
resp = HTTPResponse(
1905+
r, preload_content=False, headers={"transfer-encoding": "chunked"}
1906+
)
1907+
with pytest.raises(ProtocolError) as ctx:
1908+
next(resp.read_chunked())
1909+
1910+
assert "chunk size line exceeded maximum allowed length" in str(ctx.value)
1911+
1912+
@pytest.mark.parametrize(
1913+
"trailer_length",
1914+
[
1915+
_MAX_CHUNK_LINE_LENGTH - 1,
1916+
_MAX_CHUNK_LINE_LENGTH,
1917+
_MAX_CHUNK_LINE_LENGTH + 1,
1918+
],
1919+
)
1920+
@pytest.mark.parametrize("line_ending", [b"\r\n", b""])
1921+
def test_chunk_trailer_line_length(
1922+
self, trailer_length: int, line_ending: bytes
1923+
) -> None:
1924+
# The line limit includes its CRLF, but not the final blank line.
1925+
trailer = b"X:" + b"x" * (trailer_length - 2 - len(line_ending)) + line_ending
1926+
fp = BytesIO(b"3\r\nfoo\r\n0\r\n" + trailer + (b"\r\n" if line_ending else b""))
1927+
r = httplib.HTTPResponse(MockSock, method="GET") # type: ignore[arg-type]
1928+
r.fp = fp # type: ignore[assignment]
1929+
resp = HTTPResponse(
1930+
r,
1931+
preload_content=False,
1932+
headers={"transfer-encoding": "chunked"},
1933+
original_response=r,
1934+
)
1935+
chunks = resp.read_chunked()
1936+
assert next(chunks) == b"foo"
1937+
1938+
if trailer_length > _MAX_CHUNK_LINE_LENGTH:
1939+
with pytest.raises(
1940+
ProtocolError,
1941+
match="Response chunk trailer line exceeded maximum allowed length",
1942+
):
1943+
next(chunks)
1944+
else:
1945+
assert list(chunks) == []
1946+
1947+
assert fp.closed
1948+
assert r.isclosed()
1949+
assert resp.closed
1950+
18931951
def test_truncated_before_chunk(self) -> None:
18941952
stream = [b"foooo", b"bbbbaaaaar"]
18951953
fp = MockChunkedNoChunks(stream)
@@ -2150,7 +2208,19 @@ def pop_current_chunk(self, amt: int = -1, till_crlf: bool = False) -> bytes:
21502208
return chunk_part
21512209

21522210
def readline(self, amt: int = -1) -> bytes:
2153-
return self.pop_current_chunk(amt, till_crlf=amt < 0)
2211+
# Emulate a real file object's readline(size): return bytes up to and
2212+
# including the first newline, or at most ``amt`` bytes when ``amt >= 0``,
2213+
# whichever comes first.
2214+
if len(self.cur_chunk) <= 0:
2215+
self.cur_chunk = self._pop_new_chunk()
2216+
line = self.cur_chunk
2217+
newline_index = line.find(b"\n")
2218+
if newline_index != -1:
2219+
line = line[: newline_index + 1]
2220+
if 0 <= amt < len(line):
2221+
line = line[:amt]
2222+
self.cur_chunk = self.cur_chunk[len(line) :]
2223+
return line
21542224

21552225
def read(self, amt: int = -1) -> bytes:
21562226
return self.pop_current_chunk(amt)
@@ -2178,6 +2248,14 @@ def _encode_chunk(self, chunk: bytes) -> bytes:
21782248
return f"{self.BAD_LENGTH_LINE}{chunk.decode()}\r\n".encode()
21792249

21802250

2251+
class MockChunkedEncodingLongChunkSizeLine(MockChunkedEncodingResponse):
2252+
def _encode_chunk(self, chunk: bytes) -> bytes:
2253+
# A chunk-size line far longer than _MAX_CHUNK_LINE_LENGTH with no
2254+
# newline, simulating a malicious server that never terminates the
2255+
# size line (memory-exhaustion attack).
2256+
return b"f" * (2**16 + 1024)
2257+
2258+
21812259
class MockChunkedEncodingWithoutCRLFOnEnd(MockChunkedEncodingResponse):
21822260
def _encode_chunk(self, chunk: bytes) -> bytes:
21832261
return "{:X}\r\n{}{}".format(

‎test/with_dummyserver/test_socketlevel.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2410,6 +2410,40 @@ def socket_handler(listener: socket.socket) -> None:
24102410

24112411
done_event.set()
24122412

2413+
def test_chunked_size_line_too_long_does_not_buffer(self) -> None:
2414+
# A malicious server can advertise chunked encoding and then send an
2415+
# unterminated chunk-size line (no newline). The streaming path must
2416+
# reject it with a bounded read instead of buffering the whole run,
2417+
# which would exhaust memory.
2418+
done_event = Event()
2419+
# Well above the 65536-byte cap, but small enough for a fast test.
2420+
run_length = 512 * 1024
2421+
2422+
def socket_handler(listener: socket.socket) -> None:
2423+
sock = listener.accept()[0]
2424+
2425+
buf = b""
2426+
while not buf.endswith(b"\r\n\r\n"):
2427+
buf += sock.recv(65536)
2428+
2429+
sock.sendall(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n")
2430+
# A chunk-size line that never terminates with a newline.
2431+
with contextlib.suppress(OSError):
2432+
sock.sendall(b"f" * run_length)
2433+
2434+
done_event.wait(5)
2435+
sock.close()
2436+
2437+
self._start_server(socket_handler)
2438+
2439+
with HTTPConnectionPool(self.host, self.port, retries=False) as pool:
2440+
r = pool.request("GET", "/", timeout=LONG_TIMEOUT, preload_content=False)
2441+
with pytest.raises(ProtocolError) as ctx:
2442+
next(r.stream(65536))
2443+
assert "chunk size line exceeded maximum allowed length" in str(ctx.value)
2444+
2445+
done_event.set()
2446+
24132447
def test_large_compressed_stream(self) -> None:
24142448
done_event = Event()
24152449
expected_total_length = 296085

0 commit comments

Comments
 (0)