|
37 | 37 | ) |
38 | 38 | from tornado.log import app_log |
39 | 39 |
|
40 | | -from typing import Dict, Any, Callable, Union, Optional |
| 40 | +from typing import Dict, Any, Callable, Optional |
41 | 41 | import typing |
42 | 42 |
|
43 | 43 | if typing.TYPE_CHECKING: |
|
48 | 48 | CR_OR_LF_RE = re.compile(b"\r|\n") |
49 | 49 |
|
50 | 50 |
|
| 51 | +class _CurlStreamingBuffer: |
| 52 | + """Delivers response body chunks from libcurl to a ``streaming_callback``. |
| 53 | +
|
| 54 | + libcurl runs its write callback from inside |
| 55 | + ``curl_multi_socket_action``, where it is not safe to run application |
| 56 | + code (which may reenter this client, or raise exceptions that libcurl |
| 57 | + would turn into transfer errors). Chunks are therefore queued here and |
| 58 | + delivered later from the `.IOLoop`. |
| 59 | +
|
| 60 | + Queuing alone would let the amount of buffered data grow with whatever |
| 61 | + the server chooses to send, which is a problem when |
| 62 | + ``decompress_response`` is in use: a "decompression bomb" lets a server |
| 63 | + turn a few kilobytes on the wire into gigabytes of decompressed data, |
| 64 | + all of which would pile up in the `.IOLoop`'s callback queue before any |
| 65 | + of it reached the application. To bound this, the transfer is paused |
| 66 | + once more than `max_buffer_size` bytes are waiting, and resumed after |
| 67 | + the callback has consumed them. |
| 68 | + """ |
| 69 | + |
| 70 | + # Pause the transfer while at least this many bytes are waiting to be |
| 71 | + # delivered. This bounds our own queue only: a paused libcurl still |
| 72 | + # finishes decoding the socket read it already has (there is no way to |
| 73 | + # tell its decoder to stop) and buffers the result internally, so the |
| 74 | + # real high-water mark is one read's worth of expansion -- roughly 16MB |
| 75 | + # for gzip at libcurl's default CURLOPT_BUFFERSIZE. libcurl limits that |
| 76 | + # buffer to 64MB and fails the transfer with CURLE_TOO_LARGE past it, |
| 77 | + # which a codec with a higher ratio than deflate's (brotli, say) will |
| 78 | + # reach. That is libcurl's call to make; we just avoid holding a copy. |
| 79 | + max_buffer_size = 1024 * 1024 |
| 80 | + |
| 81 | + def __init__( |
| 82 | + self, |
| 83 | + client: "CurlAsyncHTTPClient", |
| 84 | + curl: pycurl.Curl, |
| 85 | + callback: Callable[[bytes], Any], |
| 86 | + ) -> None: |
| 87 | + self.client = client |
| 88 | + self.io_loop = client.io_loop |
| 89 | + # Set to None in finish(); the handle may be reused after that. |
| 90 | + self.curl = curl # type: Optional[pycurl.Curl] |
| 91 | + self.callback = callback |
| 92 | + self.chunks = collections.deque() # type: Deque[bytes] |
| 93 | + self.size = 0 |
| 94 | + self.total = 0 |
| 95 | + self.body_too_large = False |
| 96 | + self.paused = False |
| 97 | + self.flush_scheduled = False |
| 98 | + |
| 99 | + def write(self, chunk: bytes) -> int: |
| 100 | + """libcurl ``WRITEFUNCTION`` callback.""" |
| 101 | + if self.total + len(chunk) > self.client.max_body_size: |
| 102 | + # max_body_size applies here as well as to a buffered response: |
| 103 | + # it is a safeguard for applications that do not impose a limit |
| 104 | + # of their own on what they do with the chunks. Writing less |
| 105 | + # than we were given makes libcurl fail the transfer. |
| 106 | + self.body_too_large = True |
| 107 | + return 0 |
| 108 | + if self.size >= self.max_buffer_size: |
| 109 | + # Tell libcurl to stop reading from the socket until flush() |
| 110 | + # resumes it. This chunk is not consumed here: libcurl holds |
| 111 | + # on to it and passes it to us again when we unpause. |
| 112 | + self.paused = True |
| 113 | + return pycurl.WRITEFUNC_PAUSE |
| 114 | + self.chunks.append(chunk) |
| 115 | + self.size += len(chunk) |
| 116 | + self.total += len(chunk) |
| 117 | + self._schedule_flush() |
| 118 | + return len(chunk) |
| 119 | + |
| 120 | + def _schedule_flush(self) -> None: |
| 121 | + if not self.flush_scheduled: |
| 122 | + self.flush_scheduled = True |
| 123 | + self.io_loop.add_callback(self.flush) |
| 124 | + |
| 125 | + def flush(self) -> None: |
| 126 | + """Pass everything we have buffered to the ``streaming_callback``.""" |
| 127 | + self.flush_scheduled = False |
| 128 | + try: |
| 129 | + while self.chunks: |
| 130 | + chunk = self.chunks.popleft() |
| 131 | + self.size -= len(chunk) |
| 132 | + self.callback(chunk) |
| 133 | + finally: |
| 134 | + if self.chunks: |
| 135 | + # The callback raised; deliver the rest (and unpause) later. |
| 136 | + self._schedule_flush() |
| 137 | + elif self.paused: |
| 138 | + self.paused = False |
| 139 | + if self.curl is not None: |
| 140 | + try: |
| 141 | + # Note that this may call write() again |
| 142 | + # (synchronously) with data libcurl buffered while |
| 143 | + # paused, so it must come after our own buffer has |
| 144 | + # been drained. |
| 145 | + self.curl.pause(pycurl.PAUSE_CONT) |
| 146 | + except pycurl.error: |
| 147 | + # The transfer failed while it was paused (libcurl |
| 148 | + # reports the pending error here). Nothing to do: |
| 149 | + # _finish will pass the error to the callback. |
| 150 | + pass |
| 151 | + else: |
| 152 | + # Ask for a socket_action so the unpaused transfer |
| 153 | + # makes progress. libcurl has notified us of this |
| 154 | + # itself (through the timer callback) since 7.69, |
| 155 | + # which is below the version we require, so this is |
| 156 | + # redundant there and costs one timeout per |
| 157 | + # max_buffer_size of body. But without it a libcurl |
| 158 | + # older than that stalls until unrelated socket |
| 159 | + # activity happens to wake the transfer, and a |
| 160 | + # streaming fetch that used to work would hang. |
| 161 | + self.client._set_timeout(0) |
| 162 | + |
| 163 | + def finish(self) -> None: |
| 164 | + """Deliver any remaining data at the end of the request. |
| 165 | +
|
| 166 | + The curl handle may be reset and reused after this, so we must not |
| 167 | + touch it again. |
| 168 | + """ |
| 169 | + self.curl = None |
| 170 | + self.paused = False |
| 171 | + self.flush() |
| 172 | + |
| 173 | + |
51 | 174 | class CurlAsyncHTTPClient(AsyncHTTPClient): |
52 | 175 | def initialize( # type: ignore |
53 | | - self, max_clients: int = 10, defaults: Optional[Dict[str, Any]] = None |
| 176 | + self, |
| 177 | + max_clients: int = 10, |
| 178 | + defaults: Optional[Dict[str, Any]] = None, |
| 179 | + max_body_size: int = 104857600, |
54 | 180 | ) -> None: |
55 | 181 | super().initialize(defaults=defaults) |
| 182 | + self.max_body_size = max_body_size |
56 | 183 | # Typeshed is incomplete for CurlMulti, so just use Any for now. |
57 | 184 | self._multi = pycurl.CurlMulti() # type: Any |
58 | 185 | self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout) |
@@ -203,6 +330,8 @@ def _process_queue(self) -> None: |
203 | 330 | curl.info = { # type: ignore |
204 | 331 | "headers": httputil.HTTPHeaders(), |
205 | 332 | "buffer": BytesIO(), |
| 333 | + "streaming_buffer": None, |
| 334 | + "body_too_large": False, |
206 | 335 | "request": request, |
207 | 336 | "callback": callback, |
208 | 337 | "queue_start_time": queue_start_time, |
@@ -243,9 +372,23 @@ def _finish( |
243 | 372 | info = curl.info # type: ignore |
244 | 373 | curl.info = None # type: ignore |
245 | 374 | self._multi.remove_handle(curl) |
| 375 | + streaming_buffer = info["streaming_buffer"] |
| 376 | + body_too_large = info["body_too_large"] |
| 377 | + if streaming_buffer is not None: |
| 378 | + body_too_large = body_too_large or streaming_buffer.body_too_large |
| 379 | + # Make sure everything that was received reaches the |
| 380 | + # streaming_callback before the final response. |
| 381 | + try: |
| 382 | + streaming_buffer.finish() |
| 383 | + except Exception: |
| 384 | + self.handle_callback_exception(info["request"].streaming_callback) |
246 | 385 | buffer = info["buffer"] |
247 | 386 | if curl_error: |
248 | 387 | assert curl_message is not None |
| 388 | + if body_too_large: |
| 389 | + curl_message = "body exceeded max_body_size of %d bytes" % ( |
| 390 | + self.max_body_size, |
| 391 | + ) |
249 | 392 | error = CurlError(curl_error, curl_message) # type: Optional[CurlError] |
250 | 393 | assert error is not None |
251 | 394 | code = error.code |
@@ -344,15 +487,31 @@ def _curl_setup_request( |
344 | 487 | self._curl_header_callback, headers, request.header_callback |
345 | 488 | ), |
346 | 489 | ) |
| 490 | + write_function: Callable[[bytes], int] |
347 | 491 | if request.streaming_callback: |
348 | 492 |
|
349 | | - def write_function(b: Union[bytes, bytearray]) -> int: |
350 | | - assert request.streaming_callback is not None |
351 | | - self.io_loop.add_callback(request.streaming_callback, b) |
352 | | - return len(b) |
353 | | - |
| 493 | + streaming_buffer = _CurlStreamingBuffer( |
| 494 | + self, curl, request.streaming_callback |
| 495 | + ) |
| 496 | + curl.info["streaming_buffer"] = streaming_buffer # type: ignore |
| 497 | + write_function = streaming_buffer.write |
354 | 498 | else: |
355 | | - write_function = buffer.write # type: ignore |
| 499 | + # Count the buffered body against max_body_size, as |
| 500 | + # _CurlStreamingBuffer does for the streaming case. Here the |
| 501 | + # whole body is held in memory, so a response that decompresses |
| 502 | + # past the limit cannot be retrieved at all -- only refused |
| 503 | + # before it exhausts memory. libcurl's CURLOPT_MAXFILESIZE is |
| 504 | + # no help: outside of very recent versions it is measured |
| 505 | + # against the compressed size, which a bomb keeps small. |
| 506 | + def write_function(chunk: bytes) -> int: |
| 507 | + if buffer.tell() + len(chunk) > self.max_body_size: |
| 508 | + curl.info["body_too_large"] = True # type: ignore |
| 509 | + # Writing less than we were given makes libcurl fail |
| 510 | + # the transfer with CURLE_WRITE_ERROR; _finish uses the |
| 511 | + # flag above to replace its message with ours. |
| 512 | + return 0 |
| 513 | + return buffer.write(chunk) |
| 514 | + |
356 | 515 | curl.setopt(pycurl.WRITEFUNCTION, write_function) |
357 | 516 | curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects) |
358 | 517 | curl.setopt(pycurl.MAXREDIRS, request.max_redirects) |
|
0 commit comments