Skip to content

Commit c06bcb4

Browse files
gyanu2507elprans
andauthored
Bound close(timeout) around a pending cancel (MagicStack#1361)
close() awaited cancel_waiter with no timeout, and connection_lost only resolved waiter. A frozen server left close() hanging even after the socket dropped. Fixes MagicStack#1356 Co-authored-by: Elvis Pranskevichus <[email protected]>
1 parent 87bcaf9 commit c06bcb4

3 files changed

Lines changed: 201 additions & 32 deletions

File tree

‎asyncpg/protocol/protocol.pxd‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ cdef class BaseProtocol(CoreProtocol):
7070
cdef _on_result__copy_in(self, object waiter)
7171

7272
cdef _handle_waiter_on_connection_lost(self, cause)
73+
cdef _complete_cancel_waiters(self)
7374

7475
cdef _dispatch_result(self)
7576

‎asyncpg/protocol/protocol.pyx‎

Lines changed: 59 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -594,7 +594,15 @@ cdef class BaseProtocol(CoreProtocol):
594594
return not self.closing and self.con_status == CONNECTION_OK
595595

596596
def abort(self):
597+
# Always finish pending cancel waiters. close() sets closing=True
598+
# before awaiting them, so a later abort() must still unblock
599+
# those futures and drop the transport.
600+
self._complete_cancel_waiters()
597601
if self.closing:
602+
if self.transport is not None:
603+
transport = self.transport
604+
self.transport = None
605+
transport.abort()
598606
return
599607
self.closing = True
600608
self._handle_waiter_on_connection_lost(None)
@@ -607,40 +615,41 @@ cdef class BaseProtocol(CoreProtocol):
607615
return
608616

609617
self.closing = True
610-
611-
if self.cancel_sent_waiter is not None:
612-
await self.cancel_sent_waiter
613-
self.cancel_sent_waiter = None
614-
615-
if self.cancel_waiter is not None:
616-
await self.cancel_waiter
617-
618-
if self.waiter is not None:
619-
# If there is a query running, cancel it
620-
self._request_cancel()
621-
await self.cancel_sent_waiter
622-
self.cancel_sent_waiter = None
623-
if self.cancel_waiter is not None:
624-
await self.cancel_waiter
625-
626-
assert self.waiter is None
627-
628-
timeout = self._get_timeout_impl(timeout)
629-
630-
# Ask the server to terminate the connection and wait for it
631-
# to drop.
632-
self.waiter = self._new_waiter(timeout)
633-
self._terminate()
618+
close_waiter = None
634619
try:
635-
await self.waiter
636-
except ConnectionResetError:
637-
# There appears to be a difference in behaviour of asyncio
638-
# in Windows, where, instead of calling protocol.connection_lost()
639-
# a ConnectionResetError will be thrown into the task.
640-
pass
620+
timeout = self._get_timeout_impl(timeout)
621+
# Cancellation and the final disconnect share one deadline.
622+
async with compat.timeout(timeout):
623+
await self._drain_cancels()
624+
625+
# Transport loss completes the cancellation futures too.
626+
# There will be no further disconnect notification to await.
627+
if self.con_status != CONNECTION_OK:
628+
return
629+
630+
assert self.waiter is None
631+
632+
# The timeout context also covers errors while sending
633+
# Terminate; do not start a separate query timeout timer.
634+
close_waiter = self._new_waiter(None)
635+
self._terminate()
636+
try:
637+
await close_waiter
638+
except ConnectionResetError:
639+
# On Windows the transport may raise this instead of
640+
# calling protocol.connection_lost().
641+
pass
641642
finally:
642-
self.waiter = None
643-
self.transport.abort()
643+
if close_waiter is not None and not close_waiter.done():
644+
close_waiter.cancel()
645+
if self.timeout_handle is not None:
646+
self.timeout_handle.cancel()
647+
self.timeout_handle = None
648+
self._handle_waiter_on_connection_lost(None)
649+
if self.transport is not None:
650+
transport = self.transport
651+
self.transport = None
652+
transport.abort()
644653

645654
def _request_cancel(self):
646655
self.cancel_waiter = self.create_future()
@@ -689,6 +698,22 @@ cdef class BaseProtocol(CoreProtocol):
689698
def _create_future_fallback(self):
690699
return asyncio.Future(loop=self.loop)
691700

701+
cdef _complete_cancel_waiters(self):
702+
if (self.cancel_sent_waiter is not None and
703+
not self.cancel_sent_waiter.done()):
704+
self.cancel_sent_waiter.set_result(None)
705+
self.cancel_sent_waiter = None
706+
if self.cancel_waiter is not None and not self.cancel_waiter.done():
707+
self.cancel_waiter.set_result(None)
708+
self.cancel_waiter = None
709+
710+
async def _drain_cancels(self):
711+
await self._wait_for_cancellation()
712+
if self.waiter is not None:
713+
# If there is a query running, cancel it
714+
self._request_cancel()
715+
await self._wait_for_cancellation()
716+
692717
cdef _handle_waiter_on_connection_lost(self, cause):
693718
if self.waiter is not None and not self.waiter.done():
694719
exc = apg_exc.ConnectionDoesNotExistError(
@@ -698,6 +723,7 @@ cdef class BaseProtocol(CoreProtocol):
698723
exc.__cause__ = cause
699724
self.waiter.set_exception(exc)
700725
self.waiter = None
726+
self._complete_cancel_waiters()
701727

702728
cdef _set_server_parameter(self, name, val):
703729
self.settings.add_setting(name, val)
@@ -943,6 +969,7 @@ cdef class BaseProtocol(CoreProtocol):
943969
else:
944970
self.waiter.set_exception(exc)
945971
self.waiter = None
972+
self._complete_cancel_waiters()
946973
else:
947974
# The connection was lost because it was
948975
# terminated or due to another error;

‎tests/test_timeout.py‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,11 @@
66

77

88
import asyncio
9+
import contextlib
10+
from unittest import mock
911

1012
import asyncpg
13+
from asyncpg import connect_utils
1114
from asyncpg import connection as pg_connection
1215
from asyncpg import _testbase as tb
1316

@@ -152,3 +155,141 @@ async def test_timeout_covers_prepare_01(self):
152155
with self.assertRaises(asyncio.TimeoutError):
153156
meth = getattr(self.con, methname)
154157
await meth('select pg_sleep($1)', 0.2)
158+
159+
160+
class TestCloseTimeoutPendingCancel(tb.ClusterTestCase):
161+
162+
@contextlib.asynccontextmanager
163+
async def pending_cancel(self, *, cancel_sent, command_timeout=None):
164+
con = await self.connect(command_timeout=command_timeout)
165+
cancel_started = asyncio.Event()
166+
167+
async def cancel(**kwargs):
168+
cancel_started.set()
169+
if not cancel_sent:
170+
await self.loop.create_future()
171+
172+
# Exercise the actual Cython futures: either the cancel connection
173+
# stalls, or it closes without the query receiving ReadyForQuery.
174+
with mock.patch.object(connect_utils, '_cancel', cancel):
175+
cancellations = []
176+
try:
177+
with self.assertRaises(asyncio.TimeoutError):
178+
await con.execute('select pg_sleep(10)', timeout=0.01)
179+
await cancel_started.wait()
180+
cancellations = list(con._cancellations)
181+
self.assertTrue(con._protocol._is_cancelling())
182+
yield con
183+
finally:
184+
con.terminate()
185+
con._transport.abort()
186+
await asyncio.gather(*cancellations, return_exceptions=True)
187+
188+
async def test_close_times_out_pending_cancel(self):
189+
for cancel_sent in (False, True):
190+
with self.subTest(cancel_sent=cancel_sent):
191+
async with self.pending_cancel(cancel_sent=cancel_sent) as con:
192+
proto = con._protocol
193+
with self.assertRaises(asyncio.TimeoutError), \
194+
self.assertRunUnder(MAX_RUNTIME):
195+
await con.close(timeout=0.05)
196+
self.assertTrue(con._transport.is_closing())
197+
self.assertFalse(proto._is_cancelling())
198+
199+
async def test_close_uses_command_timeout(self):
200+
async with self.pending_cancel(
201+
cancel_sent=False, command_timeout=0.05) as con:
202+
with self.assertRaises(asyncio.TimeoutError), \
203+
self.assertRunUnder(MAX_RUNTIME):
204+
await con.close()
205+
self.assertTrue(con._transport.is_closing())
206+
207+
async def test_cancel_close_aborts_transport(self):
208+
for cancel_sent in (False, True):
209+
with self.subTest(cancel_sent=cancel_sent):
210+
async with self.pending_cancel(cancel_sent=cancel_sent) as con:
211+
task = self.loop.create_task(con.close())
212+
await asyncio.sleep(0)
213+
task.cancel()
214+
with self.assertRaises(asyncio.CancelledError):
215+
await task
216+
self.assertTrue(con._transport.is_closing())
217+
218+
async def test_connection_lost_during_close(self):
219+
for cancel_sent in (False, True):
220+
for timeout in (None, 0.05):
221+
with self.subTest(cancel_sent=cancel_sent, timeout=timeout):
222+
async with self.pending_cancel(
223+
cancel_sent=cancel_sent) as con:
224+
task = self.loop.create_task(
225+
con.close(timeout=timeout))
226+
await asyncio.sleep(0)
227+
con._transport.abort()
228+
await asyncio.wait_for(task, MAX_RUNTIME)
229+
self.assertTrue(con.is_closed())
230+
self.assertFalse(con._protocol._is_cancelling())
231+
# A lost transport must not leave a new shutdown
232+
# timer that attempts another cancel after cleanup.
233+
await asyncio.sleep(0.1)
234+
self.assertFalse(con._cancellations)
235+
236+
async def test_close_has_one_timeout_budget(self):
237+
terminate_sent = asyncio.Event()
238+
239+
class NoTerminateProtocol(connect_utils.protocol.Protocol):
240+
def connection_made(self, transport):
241+
def write(data):
242+
if bytes(data) == b'X\x00\x00\x00\x04':
243+
terminate_sent.set()
244+
else:
245+
transport.write(data)
246+
247+
# Keep the server connected after Terminate. Pausing reads
248+
# does not suppress disconnects on Windows' Proactor loop.
249+
wrapped = mock.Mock(wraps=transport)
250+
wrapped.write.side_effect = write
251+
super().connection_made(wrapped)
252+
253+
with mock.patch.object(connect_utils.protocol, 'Protocol',
254+
NoTerminateProtocol):
255+
con = await self.connect()
256+
proto = con._protocol
257+
drain_cancels = proto._drain_cancels
258+
259+
async def delayed_drain():
260+
await asyncio.sleep(0.3)
261+
await drain_cancels()
262+
263+
try:
264+
with self.assertRaises(asyncio.TimeoutError):
265+
await con.execute('select pg_sleep(10)', timeout=0.01)
266+
with mock.patch.object(proto, '_drain_cancels', delayed_drain):
267+
with self.assertRaises(asyncio.TimeoutError), \
268+
self.assertRunUnder(0.7):
269+
await con.close(timeout=0.5)
270+
self.assertTrue(terminate_sent.is_set())
271+
self.assertTrue(con._transport.is_closing())
272+
finally:
273+
con.terminate()
274+
con._transport.abort()
275+
276+
async def test_close_timeout_resolves_running_query(self):
277+
con = await self.connect()
278+
279+
async def cancel(**kwargs):
280+
await self.loop.create_future()
281+
282+
with mock.patch.object(connect_utils, '_cancel', cancel):
283+
task = self.loop.create_task(con.execute('select pg_sleep(10)'))
284+
try:
285+
await asyncio.sleep(0)
286+
with self.assertRaises(asyncio.TimeoutError):
287+
await con.close(timeout=0.05)
288+
with self.assertRaises(asyncpg.ConnectionDoesNotExistError):
289+
await asyncio.wait_for(task, MAX_RUNTIME)
290+
self.assertTrue(con._transport.is_closing())
291+
finally:
292+
con.terminate()
293+
con._transport.abort()
294+
task.cancel()
295+
await asyncio.gather(task, return_exceptions=True)

0 commit comments

Comments
 (0)