Skip to content

Commit 905ca61

Browse files
authored
[serve] Prevent ASGI receive() from hanging on disconnects (#44647)
See #44644 for the user-reported issue. In #43713, I changed the behavior of the ASGI receive interface to not raise an exception but rather silently hang if the request was not present in the proxy. This was done under the assumption that the code calling receive would be subsequently cancelled anyways. However, this assumption was faulty for a few reasons: - The user may ignore the CancelledError but still want to fetch ASGI messages. - Cancellation is best-effort. Specifically, because we are currently using the run_coroutine_threadsafe / concurrent.futures.Executor interface, there is no way to guarantee that either the scheduling task is cancelled or the resulting object ref is cancelled. In rare cases, neither may happen (see asyncio.run_coroutine_threadsafe leaves underlying cancelled asyncio task running python/cpython#105836). This PR mitigates the issue by returning an appropriate disconnect ASGI message when the proxy no longer has the request in scope. Note that this is not a perfect solution: we may still drop the messages prior to the disconnect (e.g., the body of the HTTP request) if the replica does not fetch them prior to the client disconnecting. - Note that this is not a regression from the current behavior and should only happen in practice if the client disconnects very quickly after initiating the request. - As a follow-up, we should rework this codepath to something more robust. For example we could use a push-based model where the proxy instead pushes messages eagerly to the replica (this was originally very difficult to implement but should be possible now). --------- Signed-off-by: Edward Oakes <[email protected]>
1 parent 7364705 commit 905ca61

3 files changed

Lines changed: 60 additions & 14 deletions

File tree

‎python/ray/serve/_private/http_util.py‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,14 +217,34 @@ class ASGIReceiveProxy:
217217

218218
def __init__(
219219
self,
220+
scope: Scope,
220221
request_id: str,
221222
receive_asgi_messages: Callable[[str], Awaitable[bytes]],
222223
):
224+
self._type = scope["type"] # Either 'http' or 'websocket'.
223225
self._queue = asyncio.Queue()
224226
self._request_id = request_id
225227
self._receive_asgi_messages = receive_asgi_messages
226228
self._disconnect_message = None
227229

230+
def _get_default_disconnect_message(self) -> Message:
231+
"""Return the appropriate disconnect message based on the connection type.
232+
233+
HTTP ASGI spec:
234+
https://asgi.readthedocs.io/en/latest/specs/www.html#disconnect-receive-event
235+
236+
WS ASGI spec:
237+
https://asgi.readthedocs.io/en/latest/specs/www.html#disconnect-receive-event-ws
238+
"""
239+
if self._type == "websocket":
240+
return {
241+
"type": "websocket.disconnect",
242+
# 1005 is the default disconnect code according to the ASGI spec.
243+
"code": 1005,
244+
}
245+
else:
246+
return {"type": "http.disconnect"}
247+
228248
async def fetch_until_disconnect(self):
229249
"""Fetch messages repeatedly until a disconnect message is received.
230250
@@ -244,8 +264,11 @@ async def fetch_until_disconnect(self):
244264
return
245265
except KeyError:
246266
# KeyError can be raised if the request is no longer active in the proxy
247-
# (e.g., the user disconnects). This is expected behavior and we should
267+
# (i.e., the user disconnects). This is expected behavior and we should
248268
# not log an error: https://github.com/ray-project/ray/issues/43290.
269+
message = self._get_default_disconnect_message()
270+
self._queue.put_nowait(message)
271+
self._disconnect_message = message
249272
return
250273
except Exception as e:
251274
# Raise unexpected exceptions in the next `__call__`.

‎python/ray/serve/_private/replica.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -351,6 +351,8 @@ def _wrap_user_method_call(self, request_metadata: RequestMetadata):
351351
try:
352352
self._metrics_manager.inc_num_ongoing_requests()
353353
yield
354+
except asyncio.CancelledError as e:
355+
user_exception = e
354356
except Exception as e:
355357
user_exception = e
356358
logger.error(f"Request failed:\n{e}")
@@ -942,15 +944,17 @@ def _prepare_args_for_http_request(
942944
943945
The returned `receive_task` should be cancelled when the user method exits.
944946
"""
947+
scope = pickle.loads(request.pickled_asgi_scope)
945948
receive = ASGIReceiveProxy(
949+
scope,
946950
request_metadata.request_id,
947951
request.receive_asgi_messages,
948952
)
949953
receive_task = self._user_code_event_loop.create_task(
950954
receive.fetch_until_disconnect()
951955
)
952956
asgi_args = ASGIArgs(
953-
scope=pickle.loads(request.pickled_asgi_scope),
957+
scope=scope,
954958
receive=receive,
955959
send=generator_result_callback,
956960
)

‎python/ray/serve/tests/unit/test_http_util.py‎

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import asyncio
22
import pickle
3+
import sys
34
from typing import Generator, Tuple
45

56
import pytest
@@ -65,9 +66,12 @@ async def test_message_queue():
6566

6667
@pytest.fixture
6768
@pytest.mark.asyncio
68-
def setup_receive_proxy() -> Generator[
69-
Tuple[ASGIReceiveProxy, MessageQueue], None, None
70-
]:
69+
def setup_receive_proxy(
70+
request,
71+
) -> Generator[Tuple[ASGIReceiveProxy, MessageQueue], None, None]:
72+
# Param can be 'http' (default) or 'websocket' (ASGI scope type).
73+
type = getattr(request, "param", "http")
74+
7175
queue = MessageQueue()
7276

7377
async def receive_asgi_messages(request_id: str) -> bytes:
@@ -80,7 +84,7 @@ async def receive_asgi_messages(request_id: str) -> bytes:
8084
return pickle.dumps(messages)
8185

8286
loop = get_or_create_event_loop()
83-
asgi_receive_proxy = ASGIReceiveProxy("", receive_asgi_messages)
87+
asgi_receive_proxy = ASGIReceiveProxy({"type": type}, "", receive_asgi_messages)
8488
receiver_task = loop.create_task(asgi_receive_proxy.fetch_until_disconnect())
8589
try:
8690
yield asgi_receive_proxy, queue
@@ -125,9 +129,20 @@ async def test_raises_exception(
125129
with pytest.raises(RuntimeError, match="oopsies"):
126130
await asgi_receive_proxy()
127131

128-
async def test_does_not_raise_key_error(
132+
@pytest.mark.parametrize(
133+
"setup_receive_proxy",
134+
["http", "websocket"],
135+
indirect=True,
136+
)
137+
async def test_return_disconnect_on_key_error(
129138
self, setup_receive_proxy: Tuple[ASGIReceiveProxy, MessageQueue]
130139
):
140+
"""If the proxy is no longer handling a given request, it raises a KeyError.
141+
142+
In these cases, the ASGI receive proxy should return a disconnect message.
143+
144+
See https://github.com/ray-project/ray/pull/44647 for details.
145+
"""
131146
asgi_receive_proxy, queue = setup_receive_proxy
132147

133148
queue.put_nowait({"type": "foo"})
@@ -136,17 +151,23 @@ async def test_does_not_raise_key_error(
136151
assert await asgi_receive_proxy() == {"type": "bar"}
137152

138153
queue.put_nowait(KeyError("not found"))
139-
_, pending = await asyncio.wait(
140-
[asyncio.create_task(asgi_receive_proxy())], timeout=0.01
141-
)
142-
assert len(pending) == 1
154+
for _ in range(100):
155+
if asgi_receive_proxy._type == "http":
156+
assert await asgi_receive_proxy() == {"type": "http.disconnect"}
157+
else:
158+
assert await asgi_receive_proxy() == {
159+
"type": "websocket.disconnect",
160+
"code": 1005,
161+
}
143162

144163
async def test_receive_asgi_messages_raises(self):
145164
async def receive_asgi_messages(request_id: str) -> bytes:
146165
raise RuntimeError("maybe actor crashed")
147166

148167
loop = get_or_create_event_loop()
149-
asgi_receive_proxy = ASGIReceiveProxy("", receive_asgi_messages)
168+
asgi_receive_proxy = ASGIReceiveProxy(
169+
{"type": "http"}, "", receive_asgi_messages
170+
)
150171
receiver_task = loop.create_task(asgi_receive_proxy.fetch_until_disconnect())
151172

152173
try:
@@ -157,6 +178,4 @@ async def receive_asgi_messages(request_id: str) -> bytes:
157178

158179

159180
if __name__ == "__main__":
160-
import sys
161-
162181
sys.exit(pytest.main(["-v", "-s", __file__]))

0 commit comments

Comments
 (0)