Fix HTTPResponse.read_chunked when leftover data is present in decoder's buffer (#3736)

illia-v · Cycloctane · web-flow · commit 571a9b7b894c · 2025-12-11T16:14:30.000+02:00
* Fix `HTTPResponse.read_chunked`when data is present in decoder's buffer * Simplify solution based on #3735 Co-authored-by: Rui Xi <Cycloctane@outlook.com> --------- Co-authored-by: Rui Xi <Cycloctane@outlook.com>
diff --git a/changelog/3734.bugfix.rst b/changelog/3734.bugfix.rst
@@ -0,0 +1,2 @@
+Fixed ``HTTPResponse.read_chunked()`` to properly handle leftover data in
+the decoder's buffer when reading compressed chunked responses.
diff --git a/src/urllib3/response.py b/src/urllib3/response.py
@@ -1407,10 +1407,14 @@ def read_chunked(
                 amt = None
 
             while True:
-                self._update_chunk_length()
-                if self.chunk_left == 0:
-                    break
-                chunk = self._handle_chunk(amt)
+                # First, check if any data is left in the decoder's buffer.
+                if self._decoder and self._decoder.has_unconsumed_tail:
+                    chunk = b""
+                else:
+                    self._update_chunk_length()
+                    if self.chunk_left == 0:
+                        break
+                    chunk = self._handle_chunk(amt)
                 decoded = self._decode(
                     chunk,
                     decode_content=decode_content,
diff --git a/test/test_response.py b/test/test_response.py
@@ -688,6 +688,47 @@ def test_read_with_all_data_already_in_decompressor(
         assert read(0) == b""
         assert read() == b""
 
+    @pytest.mark.parametrize("read_method", ("read_chunked", "stream"))
+    @pytest.mark.parametrize(
+        "data",
+        [d[1] for d in _test_compressor_params],
+        ids=[d[0] for d in _test_compressor_params],
+    )
+    def test_read_chunked_with_all_data_already_in_decompressor(
+        self,
+        request: pytest.FixtureRequest,
+        read_method: str,
+        data: tuple[str, typing.Callable[[bytes], bytes]] | None,
+    ) -> None:
+        if data is None:
+            pytest.skip(f"Proper {request.node.callspec.id} decoder is not available")
+        original_data = b"0" * 100_000  # 100 KB
+        name, compress_func = data
+        compressed_data = compress_func(original_data)
+        httplib_r = httplib.HTTPResponse(MockSock)  # type: ignore[arg-type]
+        httplib_r.fp = MockChunkedEncodingResponse([compressed_data])  # type: ignore[assignment]
+        r = HTTPResponse(
+            httplib_r,
+            preload_content=False,
+            headers={"transfer-encoding": "chunked", "content-encoding": name},
+        )
+        # Mark the current chunk as fully read.
+        r.chunk_left = 0
+        # Feed all compressed data to the decoder.
+        r._init_decoder()
+        assert r._decoder is not None  # for mypy
+        initially_decoded = r._decoder.decompress(compressed_data, max_length=0)
+        result = list(getattr(r, read_method)(amt=10240, decode_content=True))
+        if name == "br":
+            # It's known that some Brotli libraries do not respect
+            # `max_length`.
+            result = [initially_decoded] + list(result)
+        else:
+            assert initially_decoded == b""
+            assert all(len(chunk) == 10240 for chunk in result[:-1])
+            assert len(result[-1]) == len(original_data) % 10240
+        assert b"".join(result) == original_data
+
     @pytest.mark.parametrize(
         "delta",
         (

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	+Fixed ``HTTPResponse.read_chunked()`` to properly handle leftover data in
	`2`	`+the decoder's buffer when reading compressed chunked responses.`