Skip to content

Commit 4cf0220

Browse files
authored
Prevent stuck cancellation from blocking connection teardown (#1370)
Use connection command timeout budget when waiting for cancellation and abort the connection if cancellation cannot finish in time. Fixes: #1326 Closes: #1349
1 parent dd05865 commit 4cf0220

8 files changed

Lines changed: 270 additions & 127 deletions

File tree

‎asyncpg/connect_utils.py‎

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1310,29 +1310,31 @@ def connection_lost(self, exc):
13101310
if not self.on_disconnect.done():
13111311
self.on_disconnect.set_result(True)
13121312

1313-
if isinstance(addr, str):
1314-
tr, pr = await loop.create_unix_connection(CancelProto, addr)
1315-
else:
1316-
if params.ssl and params.sslmode != SSLMode.allow:
1317-
tr, pr = await _create_ssl_connection(
1318-
CancelProto,
1319-
*addr,
1320-
loop=loop,
1321-
ssl_context=params.ssl,
1322-
ssl_is_advisory=params.sslmode == SSLMode.prefer)
1313+
tr = None
1314+
try:
1315+
if isinstance(addr, str):
1316+
tr, pr = await loop.create_unix_connection(CancelProto, addr)
13231317
else:
1324-
tr, pr = await loop.create_connection(
1325-
CancelProto, *addr)
1326-
_set_nodelay(_get_socket(tr))
1318+
if params.ssl and params.sslmode != SSLMode.allow:
1319+
tr, pr = await _create_ssl_connection(
1320+
CancelProto,
1321+
*addr,
1322+
loop=loop,
1323+
ssl_context=params.ssl,
1324+
ssl_is_advisory=params.sslmode == SSLMode.prefer)
1325+
else:
1326+
tr, pr = await loop.create_connection(
1327+
CancelProto, *addr)
1328+
_set_nodelay(_get_socket(tr))
13271329

1328-
# Pack a CancelRequest message
1329-
msg = struct.pack('!llll', 16, 80877102, backend_pid, backend_secret)
1330+
# Pack a CancelRequest message
1331+
msg = struct.pack('!llll', 16, 80877102, backend_pid, backend_secret)
13301332

1331-
try:
13321333
tr.write(msg)
13331334
await pr.on_disconnect
13341335
finally:
1335-
tr.close()
1336+
if tr is not None:
1337+
tr.close()
13361338

13371339

13381340
def _get_socket(transport):

‎asyncpg/connection.py‎

Lines changed: 33 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1520,7 +1520,7 @@ async def close(self, *, timeout=None):
15201520

15211521
def terminate(self):
15221522
"""Terminate the connection without waiting for pending data."""
1523-
if not self.is_closed():
1523+
if not self._aborted and self._protocol is not None:
15241524
self._abort()
15251525
self._cleanup()
15261526

@@ -1573,8 +1573,9 @@ async def reset(self, *, timeout=None):
15731573
def _abort(self):
15741574
# Put the connection into the aborted state.
15751575
self._aborted = True
1576-
self._protocol.abort()
1577-
self._protocol = None
1576+
if self._protocol is not None:
1577+
self._protocol.abort()
1578+
self._protocol = None
15781579

15791580
def _cleanup(self):
15801581
self._call_termination_listeners()
@@ -1595,8 +1596,9 @@ def _cleanup(self):
15951596
def _clean_tasks(self):
15961597
# Wrap-up any remaining tasks associated with this connection.
15971598
if self._cancellations:
1599+
current = asyncio.current_task(self._loop)
15981600
for fut in self._cancellations:
1599-
if not fut.done():
1601+
if fut is not current and not fut.done():
16001602
fut.cancel()
16011603
self._cancellations.clear()
16021604

@@ -1649,37 +1651,42 @@ async def _cleanup_stmts(self):
16491651
# so we ignore the timeout.
16501652
await self._protocol.close_statement(stmt, protocol.NO_TIMEOUT)
16511653

1652-
async def _cancel(self, waiter):
1654+
async def _cancel(self, waiter, cancel_waiter=None):
16531655
try:
1654-
# Open new connection to the server
1655-
await connect_utils._cancel(
1656-
loop=self._loop, addr=self._addr, params=self._params,
1657-
backend_pid=self._protocol.backend_pid,
1658-
backend_secret=self._protocol.backend_secret)
1659-
except ConnectionResetError as ex:
1660-
# On some systems Postgres will reset the connection
1661-
# after processing the cancellation command.
1662-
if not waiter.done():
1663-
waiter.set_exception(ex)
1656+
async with compat.timeout(self._config.command_timeout):
1657+
try:
1658+
await connect_utils._cancel(
1659+
loop=self._loop, addr=self._addr, params=self._params,
1660+
backend_pid=self._protocol.backend_pid,
1661+
backend_secret=self._protocol.backend_secret)
1662+
except ConnectionResetError:
1663+
# Some servers reset the auxiliary connection after
1664+
# receiving the CancelRequest. The original connection
1665+
# still has to acknowledge the cancelled query.
1666+
pass
1667+
1668+
if not waiter.done():
1669+
waiter.set_result(None)
1670+
if cancel_waiter is not None:
1671+
await asyncio.shield(cancel_waiter)
16641672
except asyncio.CancelledError:
1665-
# There are two scenarios in which the cancellation
1666-
# itself will be cancelled: 1) the connection is being closed,
1667-
# 2) the event loop is being shut down.
1668-
# In either case we do not care about the propagation of
1669-
# the CancelledError, and don't want the loop to warn about
1670-
# an unretrieved exception.
1673+
# Teardown can cancel this background task. Its waiters are
1674+
# completed in finally, without leaking CancelledError.
16711675
pass
1672-
except (Exception, asyncio.CancelledError) as ex:
1673-
if not waiter.done():
1674-
waiter.set_exception(ex)
1676+
except Exception:
1677+
if not self._aborted:
1678+
# A failed CancelRequest leaves the original connection's
1679+
# protocol state uncertain. It cannot be reused safely.
1680+
self.terminate()
16751681
finally:
16761682
self._cancellations.discard(
16771683
asyncio.current_task(self._loop))
16781684
if not waiter.done():
16791685
waiter.set_result(None)
16801686

1681-
def _cancel_current_command(self, waiter):
1682-
self._cancellations.add(self._loop.create_task(self._cancel(waiter)))
1687+
def _cancel_current_command(self, waiter, cancel_waiter=None):
1688+
self._cancellations.add(self._loop.create_task(
1689+
self._cancel(waiter, cancel_waiter)))
16831690

16841691
def _process_log_message(self, fields, last_query):
16851692
if not self._log_listeners:

‎asyncpg/pool.py‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
import functools
1212
import inspect
1313
import logging
14-
import time
1514
from types import TracebackType
1615
from typing import Any, Optional, Type
1716
import warnings
@@ -226,12 +225,17 @@ async def release(self, timeout: Optional[float]) -> None:
226225
if self._con._protocol._is_cancelling():
227226
# If the connection is in cancellation state,
228227
# wait for the cancellation
229-
started = time.monotonic()
230-
await compat.wait_for(
231-
self._con._protocol._wait_for_cancellation(),
228+
budget = await self._con._protocol._wait_for_cancellation(
232229
budget)
233-
if budget is not None:
234-
budget -= time.monotonic() - started
230+
231+
# The background cancellation may have timed out and terminated
232+
# the connection while we were waiting. In that case cleanup
233+
# has already returned the holder to the pool.
234+
if self._con is None:
235+
return
236+
if self._con.is_closed():
237+
self._con.terminate()
238+
return
235239

236240
if self._pool._reset is not None:
237241
async with compat.timeout(budget):
@@ -246,7 +250,8 @@ async def release(self, timeout: Optional[float]) -> None:
246250
try:
247251
# An exception in `reset` is most likely caused by
248252
# an IO error, so terminate the connection.
249-
self._con.terminate()
253+
if self._con is not None:
254+
self._con.terminate()
250255
finally:
251256
raise ex
252257

‎asyncpg/protocol/protocol.pyi‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,9 @@ class BaseProtocol(CoreProtocol, Generic[_Record]):
176176
async def close(self, timeout: _TimeoutType) -> None: ...
177177
def _get_timeout(self, timeout: _TimeoutType) -> float | None: ...
178178
def _is_cancelling(self) -> bool: ...
179-
async def _wait_for_cancellation(self) -> None: ...
179+
async def _wait_for_cancellation(
180+
self, timeout: float | None = None
181+
) -> float | None: ...
180182
async def close_statement(
181183
self, state: PreparedStatementState[_OtherRecord], timeout: _TimeoutType
182184
) -> Any: ...

0 commit comments

Comments
 (0)