Skip to content

Commit ea2ad7b

Browse files
authored
Merge commit from fork
* Prevent an infinite loop when decoding Deflate responses * Move the new test to a better place * Add a changelog entry
1 parent 0716e31 commit ea2ad7b

3 files changed

Lines changed: 45 additions & 2 deletions

File tree

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fixed an infinite loop in ``HTTPResponse.stream()`` and ``HTTPResponse.read_chunked()`` when decoding chunked responses with ``Content-Encoding: deflate`` and trailing data after the compressed stream.

‎src/urllib3/response.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,10 @@ def __init__(self) -> None:
7676
def decompress(self, data: bytes, max_length: int = -1) -> bytes:
7777
data = self._unfed_data + data
7878
self._unfed_data = b""
79+
# Data received after EOF cannot produce more output from this
80+
# Deflate stream.
81+
if self._obj.eof:
82+
return b""
7983
if not data and not self._obj.unconsumed_tail:
8084
return data
8185
original_max_length = max_length
@@ -118,7 +122,9 @@ def decompress(self, data: bytes, max_length: int = -1) -> bytes:
118122
@property
119123
def has_unconsumed_tail(self) -> bool:
120124
return bool(self._unfed_data) or (
121-
bool(self._obj.unconsumed_tail) and not self._first_try
125+
bool(self._obj.unconsumed_tail)
126+
and not self._first_try
127+
and not self._obj.eof
122128
)
123129

124130
def flush(self) -> bytes:

‎test/test_response.py‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
from base64 import b64decode
1212
from http.client import IncompleteRead as httplib_IncompleteRead
1313
from io import BufferedReader, BytesIO, TextIOWrapper
14-
from test import onlyBrotli, onlyZstd
14+
from test import LONG_TIMEOUT, onlyBrotli, onlyZstd
1515
from unittest import mock
1616

1717
import pytest
@@ -813,6 +813,42 @@ def test_read_chunked_with_all_data_already_in_decompressor(
813813
assert len(result[-1]) == len(original_data) % 10240
814814
assert b"".join(result) == original_data
815815

816+
@pytest.mark.timeout(LONG_TIMEOUT)
817+
@pytest.mark.parametrize(
818+
"data",
819+
[d[1] for d in _test_compressor_params],
820+
ids=[d[0] for d in _test_compressor_params],
821+
)
822+
def test_read_chunked_with_trailing_data_does_not_hang(
823+
self,
824+
request: pytest.FixtureRequest,
825+
data: tuple[str, typing.Callable[[bytes], bytes]] | None,
826+
) -> None:
827+
if data is None:
828+
pytest.skip(f"Proper {request.node.callspec.id} decoder is not available")
829+
# The decoded body must fill multiple bounded reads so EOF is reached
830+
# while trailing data remains in the decoder's input buffer.
831+
original_data = b"A" * 100
832+
content_encoding, compress_func = data
833+
compressed_data = compress_func(original_data) + b"tail"
834+
httplib_r = httplib.HTTPResponse(MockSock) # type: ignore[arg-type]
835+
httplib_r.fp = MockChunkedEncodingResponse([compressed_data]) # type: ignore[assignment]
836+
r = HTTPResponse(
837+
httplib_r,
838+
preload_content=False,
839+
headers={
840+
"transfer-encoding": "chunked",
841+
"content-encoding": content_encoding,
842+
},
843+
)
844+
845+
stream = r.stream(len(original_data) // 2, decode_content=True)
846+
if content_encoding in ("br", "zstd"):
847+
with pytest.raises(DecodeError):
848+
list(stream)
849+
else:
850+
assert b"".join(stream) == original_data
851+
816852
@pytest.mark.parametrize(
817853
"delta",
818854
(

0 commit comments

Comments
 (0)