Skip to content

Commit ab1a778

Browse files
committed
Merge remote-tracking branch 'bdarnell/claude/asynchttpclient-streaming-memory-5fbgex-branch6.5' into fixes-659
2 parents 437ab5f + e412435 commit ab1a778

7 files changed

Lines changed: 755 additions & 47 deletions

File tree

‎docs/httpclient.rst‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ Implementations
5353

5454
.. module:: tornado.curl_httpclient
5555

56-
.. class:: CurlAsyncHTTPClient(max_clients=10, defaults=None)
56+
.. class:: CurlAsyncHTTPClient(max_clients=10, defaults=None, max_body_size=104857600)
5757

5858
``libcurl``-based HTTP client.
5959

@@ -67,6 +67,17 @@ Implementations
6767
``defaults`` is a dict of parameters that will be used as defaults on all
6868
`.HTTPRequest` objects submitted to this client.
6969

70+
``max_body_size`` (default 100MB) is the largest response body that the
71+
client will accept, measured after decompression so that a compressed
72+
response cannot expand past it. As in `.SimpleAsyncHTTPClient`, it applies
73+
whether or not a ``streaming_callback`` is used; raise it to retrieve
74+
responses larger than that.
75+
76+
.. versionchanged:: 6.5.9
77+
Added the ``max_body_size`` argument. Previously a response body was
78+
accepted without limit, so a server could exhaust the client's memory
79+
with a compressed response.
80+
7081
Example Code
7182
~~~~~~~~~~~~
7283

‎docs/index.rst‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,8 @@ source tarball or clone the `git repository
103103
optional packages may be useful:
104104

105105
* `pycurl <http://pycurl.io/>`_ is used by the optional
106-
``tornado.curl_httpclient``. Libcurl version 7.22 or higher is required.
106+
``tornado.curl_httpclient``. Libcurl version 7.81 or higher is required
107+
(the version shipped by Ubuntu 22.04 LTS).
107108
* `pycares <https://pypi.org/project/pycares/>`_ is an alternative
108109
non-blocking DNS resolver that can be used when threads are not
109110
appropriate.

‎tornado/curl_httpclient.py‎

Lines changed: 167 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
)
3838
from tornado.log import app_log
3939

40-
from typing import Dict, Any, Callable, Union, Optional
40+
from typing import Dict, Any, Callable, Optional
4141
import typing
4242

4343
if typing.TYPE_CHECKING:
@@ -48,11 +48,138 @@
4848
CR_OR_LF_RE = re.compile(b"\r|\n")
4949

5050

51+
class _CurlStreamingBuffer:
52+
"""Delivers response body chunks from libcurl to a ``streaming_callback``.
53+
54+
libcurl runs its write callback from inside
55+
``curl_multi_socket_action``, where it is not safe to run application
56+
code (which may reenter this client, or raise exceptions that libcurl
57+
would turn into transfer errors). Chunks are therefore queued here and
58+
delivered later from the `.IOLoop`.
59+
60+
Queuing alone would let the amount of buffered data grow with whatever
61+
the server chooses to send, which is a problem when
62+
``decompress_response`` is in use: a "decompression bomb" lets a server
63+
turn a few kilobytes on the wire into gigabytes of decompressed data,
64+
all of which would pile up in the `.IOLoop`'s callback queue before any
65+
of it reached the application. To bound this, the transfer is paused
66+
once more than `max_buffer_size` bytes are waiting, and resumed after
67+
the callback has consumed them.
68+
"""
69+
70+
# Pause the transfer while at least this many bytes are waiting to be
71+
# delivered. This bounds our own queue only: a paused libcurl still
72+
# finishes decoding the socket read it already has (there is no way to
73+
# tell its decoder to stop) and buffers the result internally, so the
74+
# real high-water mark is one read's worth of expansion -- roughly 16MB
75+
# for gzip at libcurl's default CURLOPT_BUFFERSIZE. libcurl limits that
76+
# buffer to 64MB and fails the transfer with CURLE_TOO_LARGE past it,
77+
# which a codec with a higher ratio than deflate's (brotli, say) will
78+
# reach. That is libcurl's call to make; we just avoid holding a copy.
79+
max_buffer_size = 1024 * 1024
80+
81+
def __init__(
82+
self,
83+
client: "CurlAsyncHTTPClient",
84+
curl: pycurl.Curl,
85+
callback: Callable[[bytes], Any],
86+
) -> None:
87+
self.client = client
88+
self.io_loop = client.io_loop
89+
# Set to None in finish(); the handle may be reused after that.
90+
self.curl = curl # type: Optional[pycurl.Curl]
91+
self.callback = callback
92+
self.chunks = collections.deque() # type: Deque[bytes]
93+
self.size = 0
94+
self.total = 0
95+
self.body_too_large = False
96+
self.paused = False
97+
self.flush_scheduled = False
98+
99+
def write(self, chunk: bytes) -> int:
100+
"""libcurl ``WRITEFUNCTION`` callback."""
101+
if self.total + len(chunk) > self.client.max_body_size:
102+
# max_body_size applies here as well as to a buffered response:
103+
# it is a safeguard for applications that do not impose a limit
104+
# of their own on what they do with the chunks. Writing less
105+
# than we were given makes libcurl fail the transfer.
106+
self.body_too_large = True
107+
return 0
108+
if self.size >= self.max_buffer_size:
109+
# Tell libcurl to stop reading from the socket until flush()
110+
# resumes it. This chunk is not consumed here: libcurl holds
111+
# on to it and passes it to us again when we unpause.
112+
self.paused = True
113+
return pycurl.WRITEFUNC_PAUSE
114+
self.chunks.append(chunk)
115+
self.size += len(chunk)
116+
self.total += len(chunk)
117+
self._schedule_flush()
118+
return len(chunk)
119+
120+
def _schedule_flush(self) -> None:
121+
if not self.flush_scheduled:
122+
self.flush_scheduled = True
123+
self.io_loop.add_callback(self.flush)
124+
125+
def flush(self) -> None:
126+
"""Pass everything we have buffered to the ``streaming_callback``."""
127+
self.flush_scheduled = False
128+
try:
129+
while self.chunks:
130+
chunk = self.chunks.popleft()
131+
self.size -= len(chunk)
132+
self.callback(chunk)
133+
finally:
134+
if self.chunks:
135+
# The callback raised; deliver the rest (and unpause) later.
136+
self._schedule_flush()
137+
elif self.paused:
138+
self.paused = False
139+
if self.curl is not None:
140+
try:
141+
# Note that this may call write() again
142+
# (synchronously) with data libcurl buffered while
143+
# paused, so it must come after our own buffer has
144+
# been drained.
145+
self.curl.pause(pycurl.PAUSE_CONT)
146+
except pycurl.error:
147+
# The transfer failed while it was paused (libcurl
148+
# reports the pending error here). Nothing to do:
149+
# _finish will pass the error to the callback.
150+
pass
151+
else:
152+
# Ask for a socket_action so the unpaused transfer
153+
# makes progress. libcurl has notified us of this
154+
# itself (through the timer callback) since 7.69,
155+
# which is below the version we require, so this is
156+
# redundant there and costs one timeout per
157+
# max_buffer_size of body. But without it a libcurl
158+
# older than that stalls until unrelated socket
159+
# activity happens to wake the transfer, and a
160+
# streaming fetch that used to work would hang.
161+
self.client._set_timeout(0)
162+
163+
def finish(self) -> None:
164+
"""Deliver any remaining data at the end of the request.
165+
166+
The curl handle may be reset and reused after this, so we must not
167+
touch it again.
168+
"""
169+
self.curl = None
170+
self.paused = False
171+
self.flush()
172+
173+
51174
class CurlAsyncHTTPClient(AsyncHTTPClient):
52175
def initialize( # type: ignore
53-
self, max_clients: int = 10, defaults: Optional[Dict[str, Any]] = None
176+
self,
177+
max_clients: int = 10,
178+
defaults: Optional[Dict[str, Any]] = None,
179+
max_body_size: int = 104857600,
54180
) -> None:
55181
super().initialize(defaults=defaults)
182+
self.max_body_size = max_body_size
56183
# Typeshed is incomplete for CurlMulti, so just use Any for now.
57184
self._multi = pycurl.CurlMulti() # type: Any
58185
self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout)
@@ -203,6 +330,8 @@ def _process_queue(self) -> None:
203330
curl.info = { # type: ignore
204331
"headers": httputil.HTTPHeaders(),
205332
"buffer": BytesIO(),
333+
"streaming_buffer": None,
334+
"body_too_large": False,
206335
"request": request,
207336
"callback": callback,
208337
"queue_start_time": queue_start_time,
@@ -243,9 +372,23 @@ def _finish(
243372
info = curl.info # type: ignore
244373
curl.info = None # type: ignore
245374
self._multi.remove_handle(curl)
375+
streaming_buffer = info["streaming_buffer"]
376+
body_too_large = info["body_too_large"]
377+
if streaming_buffer is not None:
378+
body_too_large = body_too_large or streaming_buffer.body_too_large
379+
# Make sure everything that was received reaches the
380+
# streaming_callback before the final response.
381+
try:
382+
streaming_buffer.finish()
383+
except Exception:
384+
self.handle_callback_exception(info["request"].streaming_callback)
246385
buffer = info["buffer"]
247386
if curl_error:
248387
assert curl_message is not None
388+
if body_too_large:
389+
curl_message = "body exceeded max_body_size of %d bytes" % (
390+
self.max_body_size,
391+
)
249392
error = CurlError(curl_error, curl_message) # type: Optional[CurlError]
250393
assert error is not None
251394
code = error.code
@@ -344,15 +487,31 @@ def _curl_setup_request(
344487
self._curl_header_callback, headers, request.header_callback
345488
),
346489
)
490+
write_function: Callable[[bytes], int]
347491
if request.streaming_callback:
348492

349-
def write_function(b: Union[bytes, bytearray]) -> int:
350-
assert request.streaming_callback is not None
351-
self.io_loop.add_callback(request.streaming_callback, b)
352-
return len(b)
353-
493+
streaming_buffer = _CurlStreamingBuffer(
494+
self, curl, request.streaming_callback
495+
)
496+
curl.info["streaming_buffer"] = streaming_buffer # type: ignore
497+
write_function = streaming_buffer.write
354498
else:
355-
write_function = buffer.write # type: ignore
499+
# Count the buffered body against max_body_size, as
500+
# _CurlStreamingBuffer does for the streaming case. Here the
501+
# whole body is held in memory, so a response that decompresses
502+
# past the limit cannot be retrieved at all -- only refused
503+
# before it exhausts memory. libcurl's CURLOPT_MAXFILESIZE is
504+
# no help: outside of very recent versions it is measured
505+
# against the compressed size, which a bomb keeps small.
506+
def write_function(chunk: bytes) -> int:
507+
if buffer.tell() + len(chunk) > self.max_body_size:
508+
curl.info["body_too_large"] = True # type: ignore
509+
# Writing less than we were given makes libcurl fail
510+
# the transfer with CURLE_WRITE_ERROR; _finish uses the
511+
# flag above to replace its message with ours.
512+
return 0
513+
return buffer.write(chunk)
514+
356515
curl.setopt(pycurl.WRITEFUNCTION, write_function)
357516
curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects)
358517
curl.setopt(pycurl.MAXREDIRS, request.max_redirects)

‎tornado/test/circlerefs_test.py‎

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,14 @@
2121
from tornado import web, gen, httpclient
2222
from tornado.test.util import skipNotCPython
2323

24+
try:
25+
import pycurl
26+
except ImportError:
27+
pycurl = None # type: ignore
28+
29+
if pycurl is not None:
30+
from tornado.curl_httpclient import CurlAsyncHTTPClient
31+
2432

2533
def find_circular_references(garbage):
2634
"""Find circular references in a list of objects.
@@ -100,6 +108,11 @@ def assert_no_cycle_garbage():
100108
gc.enable()
101109

102110

111+
class OkHandler(web.RequestHandler):
112+
def get(self):
113+
self.write("ok\n")
114+
115+
103116
# GC behavior is cpython-specific
104117
@skipNotCPython
105118
class CircleRefsTest(unittest.TestCase):
@@ -137,7 +150,7 @@ def __repr__(self):
137150
self.assertIn(" name=b", str(cm.exception))
138151
self.assertNotIn(" name=c", str(cm.exception))
139152

140-
async def run_handler(self, handler_class):
153+
async def run_handler(self, handler_class, client_factory=None, **fetch_kwargs):
141154
app = web.Application(
142155
[
143156
(r"/", handler_class),
@@ -147,14 +160,18 @@ async def run_handler(self, handler_class):
147160
server = tornado.httpserver.HTTPServer(app)
148161
server.add_socket(socket)
149162

150-
client = httpclient.AsyncHTTPClient()
163+
# The client must be constructed here rather than by the caller: it
164+
# binds to the IOLoop that is current when it is created.
165+
if client_factory is None:
166+
client_factory = httpclient.AsyncHTTPClient
167+
client = client_factory()
151168
with assert_no_cycle_garbage():
152169
# Only the fetch (and the corresponding server-side handler)
153170
# are being tested for cycles. In particular, the Application
154171
# object has internal cycles (as of this writing) which we don't
155172
# care to fix since in real world usage the Application object
156173
# is effectively a global singleton.
157-
await client.fetch(f"http://127.0.0.1:{port}/")
174+
await client.fetch(f"http://127.0.0.1:{port}/", **fetch_kwargs)
158175
client.close()
159176
server.stop()
160177
socket.close()
@@ -190,6 +207,33 @@ async def get(self):
190207

191208
asyncio.run(self.run_handler(Handler))
192209

210+
# The tests above vary the server-side handler against one client. These
211+
# vary the client instead: a streaming_callback takes a different path
212+
# through both client implementations than a buffered response does, and
213+
# curl_httpclient keeps per-request state on pooled curl handles.
214+
215+
def test_streaming_callback(self):
216+
asyncio.run(self.run_handler(OkHandler, streaming_callback=lambda chunk: None))
217+
218+
@unittest.skipIf(pycurl is None, "pycurl module not present")
219+
def test_curl_httpclient(self):
220+
asyncio.run(
221+
self.run_handler(
222+
OkHandler,
223+
client_factory=lambda: CurlAsyncHTTPClient(force_instance=True),
224+
)
225+
)
226+
227+
@unittest.skipIf(pycurl is None, "pycurl module not present")
228+
def test_curl_httpclient_streaming_callback(self):
229+
asyncio.run(
230+
self.run_handler(
231+
OkHandler,
232+
client_factory=lambda: CurlAsyncHTTPClient(force_instance=True),
233+
streaming_callback=lambda chunk: None,
234+
)
235+
)
236+
193237
def test_run_on_executor(self):
194238
# From https://github.com/tornadoweb/tornado/issues/2620
195239
#

0 commit comments

Comments
 (0)