Skip to content

Commit 5ff81de

Browse files
cliffhallclaude
andcommitted
fix(fetch): stop reading a compressed body where its stream ends
After a gzip or deflate stream completes, zlib keeps every further input byte in unused_data, which the output cap never sees, so a complete compressed body followed by endless trailing data still grew memory without bound, for a page and for robots.txt. Reading now stops at the end of the compressed stream; httpx's decoders drop trailing data too, so the decoded body is unchanged. Refs #5370 Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: cliffhall <[email protected]>
1 parent 781e72a commit 5ff81de

2 files changed

Lines changed: 67 additions & 0 deletions

File tree

‎src/fetch/src/mcp_server_fetch/server.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,11 @@ def __init__(self, coding: str) -> None:
435435
def unconsumed_tail(self) -> bytes:
436436
return self._z.unconsumed_tail
437437

438+
@property
439+
def eof(self) -> bool:
440+
"""Whether the compressed stream has ended."""
441+
return self._z.eof
442+
438443
def decompress(self, data: bytes, max_length: int) -> bytes:
439444
try:
440445
out = self._z.decompress(data, max_length)
@@ -536,6 +541,11 @@ async def read_capped(response: httpx.Response, limit: int) -> tuple[bytes, bool
536541
body += out
537542
if len(body) > limit:
538543
return bytes(body[:limit]), True
544+
if inflater is not None and inflater.eof:
545+
# The compressed stream is complete. Whatever follows it is not
546+
# part of the body (httpx's decoders drop it too), and zlib
547+
# would keep all of it in `unused_data`, so reading stops here.
548+
return bytes(body), False
539549
if inflater is not None:
540550
# All input is consumed by now, so zlib has at most a few bytes left.
541551
body += inflater.flush()

‎src/fetch/tests/test_response_size_cap.py‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,63 @@ def decompress(self, data: bytes, max_length: int = 0) -> bytes:
297297
assert sum(outputs) <= CAP + 1
298298

299299

300+
@pytest.mark.parametrize(
301+
("coding", "encode"),
302+
[("gzip", gzip.compress), ("deflate", zlib.compress), ("deflate", raw_deflate)],
303+
)
304+
async def test_data_after_the_compressed_stream_is_not_read(
305+
web: FakeWeb, coding: str, encode: Any
306+
) -> None:
307+
# A complete compressed body, then endless trailing bytes: zlib would hold
308+
# every one of them in `unused_data`, which no output cap sees, so the read
309+
# stops where the compressed stream ends.
310+
trailing = Body(fill=b"\x00")
311+
312+
async def stream() -> AsyncIterator[bytes]:
313+
yield encode(b"t" * 10)
314+
async for chunk in trailing.stream():
315+
yield chunk
316+
317+
web.add(
318+
PAGE,
319+
httpx.Response(
320+
200,
321+
content=stream(),
322+
headers={"content-type": "text/plain", "content-encoding": coding},
323+
),
324+
)
325+
result = await call()
326+
assert wire(result)["isError"] is False
327+
assert text_of(result).endswith("t" * 10)
328+
assert trailing.chunks_sent <= 1
329+
330+
331+
async def test_robots_txt_stops_at_the_end_of_its_compressed_stream(
332+
web: FakeWeb,
333+
) -> None:
334+
trailing = Body(fill=b"\x00")
335+
336+
async def robots() -> AsyncIterator[bytes]:
337+
yield gzip.compress(b"User-agent: *\nDisallow: /page\n")
338+
async for chunk in trailing.stream():
339+
yield chunk
340+
341+
web.add(
342+
ROBOTS,
343+
httpx.Response(
344+
200,
345+
content=robots(),
346+
headers={"content-type": "text/plain", "content-encoding": "gzip"},
347+
),
348+
)
349+
async with connect() as (session, _):
350+
result = await session.call_tool("fetch", {"url": PAGE})
351+
assert "specifies that autonomous fetching of this page is not allowed" in (
352+
text_of(result)
353+
)
354+
assert trailing.chunks_sent <= 1
355+
356+
300357
@pytest.mark.parametrize(
301358
("coding", "encode"),
302359
[

0 commit comments

Comments
 (0)