Skip to content

Commit 1c43ad6

Browse files
committed
integrate connection.terminate() for supporting dialects
Integrated support for asyncpg's ``terminate()`` method call for cases where the connection pool is recycling a possibly timed-out connection, where a connection is being garbage collected that wasn't gracefully closed, as well as when the connection has been invalidated. This allows asyncpg to abandon the connection without waiting for a response that may incur long timeouts. Fixes: #8419 Change-Id: Ia575af779d5733b483a72dff3690b8bbbad2bb05 (cherry picked from commit 3b7e621aa728d9b01dbac4150e13ea2ef6af35a3)
1 parent 3ffca71 commit 1c43ad6

7 files changed

Lines changed: 82 additions & 17 deletions

File tree

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
.. change::
2+
:tags: bug, asyncio
3+
:tickets: 8419
4+
5+
Integrated support for asyncpg's ``terminate()`` method call for cases
6+
where the connection pool is recycling a possibly timed-out connection,
7+
where a connection is being garbage collected that wasn't gracefully
8+
closed, as well as when the connection has been invalidated. This allows
9+
asyncpg to abandon the connection without waiting for a response that may
10+
incur long timeouts.

‎lib/sqlalchemy/dialects/postgresql/asyncpg.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -748,6 +748,9 @@ def close(self):
748748

749749
self.await_(self._connection.close())
750750

751+
def terminate(self):
752+
self._connection.terminate()
753+
751754

752755
class AsyncAdaptFallback_asyncpg_connection(AsyncAdapt_asyncpg_connection):
753756
__slots__ = ()
@@ -891,6 +894,7 @@ class PGDialect_asyncpg(PGDialect):
891894
supports_server_side_cursors = True
892895

893896
supports_unicode_binds = True
897+
has_terminate = True
894898

895899
default_paramstyle = "format"
896900
supports_sane_multi_rowcount = False
@@ -987,6 +991,9 @@ def set_deferrable(self, connection, value):
987991
def get_deferrable(self, connection):
988992
return connection.deferrable
989993

994+
def do_terminate(self, dbapi_connection) -> None:
995+
dbapi_connection.terminate()
996+
990997
def create_connect_args(self, url):
991998
opts = url.translate_connect_args(username="user")
992999

‎lib/sqlalchemy/engine/default.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,7 @@ class DefaultDialect(interfaces.Dialect):
231231
CACHING_DISABLED = CACHING_DISABLED
232232
NO_CACHE_KEY = NO_CACHE_KEY
233233
NO_DIALECT_SUPPORT = NO_DIALECT_SUPPORT
234+
has_terminate = False
234235

235236
@util.deprecated_params(
236237
convert_unicode=(
@@ -684,6 +685,9 @@ def do_rollback(self, dbapi_connection):
684685
def do_commit(self, dbapi_connection):
685686
dbapi_connection.commit()
686687

688+
def do_terminate(self, dbapi_connection):
689+
self.do_close(dbapi_connection)
690+
687691
def do_close(self, dbapi_connection):
688692
dbapi_connection.close()
689693

‎lib/sqlalchemy/engine/interfaces.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -583,6 +583,23 @@ def do_commit(self, dbapi_connection):
583583

584584
raise NotImplementedError()
585585

586+
def do_terminate(self, dbapi_connection):
587+
"""Provide an implementation of ``connection.close()`` that tries as
588+
much as possible to not block, given a DBAPI
589+
connection.
590+
591+
In the vast majority of cases this just calls .close(), however
592+
for some asyncio dialects may call upon different API features.
593+
594+
This hook is called by the :class:`_pool.Pool`
595+
when a connection is being recycled or has been invalidated.
596+
597+
.. versionadded:: 1.4.41
598+
599+
"""
600+
601+
raise NotImplementedError()
602+
586603
def do_close(self, dbapi_connection):
587604
"""Provide an implementation of ``connection.close()``, given a DBAPI
588605
connection.

‎lib/sqlalchemy/pool/base.py‎

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,17 @@ class _ConnDialect(object):
3636
"""
3737

3838
is_async = False
39+
has_terminate = False
3940

4041
def do_rollback(self, dbapi_connection):
4142
dbapi_connection.rollback()
4243

4344
def do_commit(self, dbapi_connection):
4445
dbapi_connection.commit()
4546

47+
def do_terminate(self, dbapi_connection):
48+
dbapi_connection.close()
49+
4650
def do_close(self, dbapi_connection):
4751
dbapi_connection.close()
4852

@@ -240,11 +244,17 @@ def _should_wrap_creator(self, creator):
240244
else:
241245
return lambda crec: creator()
242246

243-
def _close_connection(self, connection):
244-
self.logger.debug("Closing connection %r", connection)
245-
247+
def _close_connection(self, connection, terminate=False):
248+
self.logger.debug(
249+
"%s connection %r",
250+
"Hard-closing" if terminate else "Closing",
251+
connection,
252+
)
246253
try:
247-
self._dialect.do_close(connection)
254+
if terminate:
255+
self._dialect.do_terminate(connection)
256+
else:
257+
self._dialect.do_close(connection)
248258
except Exception:
249259
self.logger.error(
250260
"Exception closing connection %r", connection, exc_info=True
@@ -584,7 +594,7 @@ def invalidate(self, e=None, soft=False):
584594
if soft:
585595
self._soft_invalidate_time = time.time()
586596
else:
587-
self.__close()
597+
self.__close(terminate=True)
588598
self.dbapi_connection = None
589599

590600
def get_connection(self):
@@ -630,7 +640,7 @@ def get_connection(self):
630640
recycle = True
631641

632642
if recycle:
633-
self.__close()
643+
self.__close(terminate=True)
634644
self.info.clear()
635645

636646
self.__connect()
@@ -643,11 +653,13 @@ def _is_hard_or_soft_invalidated(self):
643653
or (self._soft_invalidate_time > self.starttime)
644654
)
645655

646-
def __close(self):
656+
def __close(self, terminate=False):
647657
self.finalize_callback.clear()
648658
if self.__pool.dispatch.close:
649659
self.__pool.dispatch.close(self.dbapi_connection, self)
650-
self.__pool._close_connection(self.dbapi_connection)
660+
self.__pool._close_connection(
661+
self.dbapi_connection, terminate=terminate
662+
)
651663
self.dbapi_connection = None
652664

653665
def __connect(self):
@@ -709,7 +721,9 @@ def _finalize_fairy(
709721
dbapi_connection = connection_record.dbapi_connection
710722

711723
# null pool is not _is_asyncio but can be used also with async dialects
712-
dont_restore_gced = pool._dialect.is_async
724+
dont_restore_gced = (
725+
pool._dialect.is_async and not pool._dialect.has_terminate
726+
)
713727

714728
if dont_restore_gced:
715729
detach = not connection_record or ref
@@ -751,8 +765,10 @@ def _finalize_fairy(
751765
else:
752766
message = (
753767
"The garbage collector is trying to clean up "
754-
"connection %r. This feature is unsupported on async "
755-
"dbapi, since no IO can be performed at this stage to "
768+
"connection %r. This feature is unsupported on "
769+
"unsupported on asyncio "
770+
'dbapis that lack a "terminate" feature, '
771+
"since no IO can be performed at this stage to "
756772
"reset the connection. Please close out all "
757773
"connections when they are no longer used, calling "
758774
"``close()`` or using a context manager to "

‎test/engine/test_logging.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -468,7 +468,7 @@ def _test_queuepool(self, q, dispose=True):
468468
"Connection %r checked out from pool",
469469
"Connection %r being returned to pool%s",
470470
"Connection %s rollback-on-return",
471-
"Closing connection %r",
471+
"%s connection %r",
472472
]
473473
+ (["Pool disposed. %s"] if dispose else []),
474474
)

‎test/engine/test_pool.py‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -92,10 +92,13 @@ def _queuepool_fixture(self, **kw):
9292
def _queuepool_dbapi_fixture(self, **kw):
9393
dbapi = MockDBAPI()
9494
_is_asyncio = kw.pop("_is_asyncio", False)
95+
_has_terminate = kw.pop("_has_terminate", False)
9596
p = pool.QueuePool(creator=lambda: dbapi.connect("foo.db"), **kw)
9697
if _is_asyncio:
9798
p._is_asyncio = True
9899
p._dialect = _AsyncConnDialect()
100+
if _has_terminate:
101+
p._dialect.has_terminate = True
99102
return dbapi, p
100103

101104

@@ -468,8 +471,10 @@ def checkout(*arg, **kw):
468471

469472
return p, canary
470473

471-
def _checkin_event_fixture(self, _is_asyncio=False):
472-
p = self._queuepool_fixture(_is_asyncio=_is_asyncio)
474+
def _checkin_event_fixture(self, _is_asyncio=False, _has_terminate=False):
475+
p = self._queuepool_fixture(
476+
_is_asyncio=_is_asyncio, _has_terminate=_has_terminate
477+
)
473478
canary = []
474479

475480
@event.listens_for(p, "checkin")
@@ -744,9 +749,13 @@ def test_invalidate_event_exception(self):
744749
assert canary.call_args_list[0][0][0] is dbapi_con
745750
assert canary.call_args_list[0][0][2] is exc
746751

747-
@testing.combinations((True, testing.requires.python3), (False,))
748-
def test_checkin_event_gc(self, detach_gced):
749-
p, canary = self._checkin_event_fixture(_is_asyncio=detach_gced)
752+
@testing.combinations((True,), (False,), argnames="is_asyncio")
753+
@testing.combinations((True,), (False,), argnames="has_terminate")
754+
@testing.requires.python3
755+
def test_checkin_event_gc(self, is_asyncio, has_terminate):
756+
p, canary = self._checkin_event_fixture(
757+
_is_asyncio=is_asyncio, _has_terminate=has_terminate
758+
)
750759

751760
c1 = p.connect()
752761

@@ -756,6 +765,8 @@ def test_checkin_event_gc(self, detach_gced):
756765
del c1
757766
lazy_gc()
758767

768+
detach_gced = is_asyncio and not has_terminate
769+
759770
if detach_gced:
760771
# "close_detached" is not called because for asyncio the
761772
# connection is just lost.

0 commit comments

Comments
 (0)