Skip to content

Commit 776abf4

Browse files
committed
integrate connection.terminate() for supporting dialects
Integrated support for asyncpg's ``terminate()`` method call for cases where the connection pool is recycling a possibly timed-out connection, where a connection is being garbage collected that wasn't gracefully closed, as well as when the connection has been invalidated. This allows asyncpg to abandon the connection without waiting for a response that may incur long timeouts. Fixes: #8419 Change-Id: Ia575af779d5733b483a72dff3690b8bbbad2bb05
1 parent 27bf1c1 commit 776abf4

7 files changed

Lines changed: 87 additions & 16 deletions

File tree

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
.. change::
2+
:tags: bug, asyncio
3+
:tickets: 8419
4+
5+
Integrated support for asyncpg's ``terminate()`` method call for cases
6+
where the connection pool is recycling a possibly timed-out connection,
7+
where a connection is being garbage collected that wasn't gracefully
8+
closed, as well as when the connection has been invalidated. This allows
9+
asyncpg to abandon the connection without waiting for a response that may
10+
incur long timeouts.

‎lib/sqlalchemy/dialects/postgresql/asyncpg.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -793,6 +793,9 @@ def close(self):
793793

794794
self.await_(self._connection.close())
795795

796+
def terminate(self):
797+
self._connection.terminate()
798+
796799

797800
class AsyncAdaptFallback_asyncpg_connection(AsyncAdapt_asyncpg_connection):
798801
__slots__ = ()
@@ -895,6 +898,7 @@ class PGDialect_asyncpg(PGDialect):
895898
supports_server_side_cursors = True
896899

897900
render_bind_cast = True
901+
has_terminate = True
898902

899903
default_paramstyle = "format"
900904
supports_sane_multi_rowcount = False
@@ -981,6 +985,9 @@ def set_deferrable(self, connection, value):
981985
def get_deferrable(self, connection):
982986
return connection.deferrable
983987

988+
def do_terminate(self, dbapi_connection) -> None:
989+
dbapi_connection.terminate()
990+
984991
def create_connect_args(self, url):
985992
opts = url.translate_connect_args(username="user")
986993

‎lib/sqlalchemy/engine/default.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,8 @@ class DefaultDialect(Dialect):
237237

238238
is_async = False
239239

240+
has_terminate = False
241+
240242
# TODO: this is not to be part of 2.0. implement rudimentary binary
241243
# literals for SQLite, PostgreSQL, MySQL only within
242244
# _Binary.literal_processor
@@ -620,6 +622,9 @@ def do_rollback(self, dbapi_connection):
620622
def do_commit(self, dbapi_connection):
621623
dbapi_connection.commit()
622624

625+
def do_terminate(self, dbapi_connection):
626+
self.do_close(dbapi_connection)
627+
623628
def do_close(self, dbapi_connection):
624629
dbapi_connection.close()
625630

‎lib/sqlalchemy/engine/interfaces.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -966,6 +966,10 @@ def loaded_dbapi(self) -> ModuleType:
966966
is_async: bool
967967
"""Whether or not this dialect is intended for asyncio use."""
968968

969+
has_terminate: bool
970+
"""Whether or not this dialect has a separate "terminate" implementation
971+
that does not block or require awaiting."""
972+
969973
engine_config_types: Mapping[str, Any]
970974
"""a mapping of string keys that can be in an engine config linked to
971975
type conversion functions.
@@ -1784,6 +1788,23 @@ def do_commit(self, dbapi_connection: PoolProxiedConnection) -> None:
17841788

17851789
raise NotImplementedError()
17861790

1791+
def do_terminate(self, dbapi_connection: DBAPIConnection) -> None:
1792+
"""Provide an implementation of ``connection.close()`` that tries as
1793+
much as possible to not block, given a DBAPI
1794+
connection.
1795+
1796+
In the vast majority of cases this just calls .close(), however
1797+
for some asyncio dialects may call upon different API features.
1798+
1799+
This hook is called by the :class:`_pool.Pool`
1800+
when a connection is being recycled or has been invalidated.
1801+
1802+
.. versionadded:: 1.4.41
1803+
1804+
"""
1805+
1806+
raise NotImplementedError()
1807+
17871808
def do_close(self, dbapi_connection: DBAPIConnection) -> None:
17881809
"""Provide an implementation of ``connection.close()``, given a DBAPI
17891810
connection.

‎lib/sqlalchemy/pool/base.py‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -72,13 +72,17 @@ class _ConnDialect:
7272
"""
7373

7474
is_async = False
75+
has_terminate = False
7576

7677
def do_rollback(self, dbapi_connection: PoolProxiedConnection) -> None:
7778
dbapi_connection.rollback()
7879

7980
def do_commit(self, dbapi_connection: PoolProxiedConnection) -> None:
8081
dbapi_connection.commit()
8182

83+
def do_terminate(self, dbapi_connection: DBAPIConnection) -> None:
84+
dbapi_connection.close()
85+
8286
def do_close(self, dbapi_connection: DBAPIConnection) -> None:
8387
dbapi_connection.close()
8488

@@ -310,10 +314,19 @@ def _should_wrap_creator(
310314
creator_fn = cast(_CreatorFnType, creator)
311315
return lambda rec: creator_fn()
312316

313-
def _close_connection(self, connection: DBAPIConnection) -> None:
314-
self.logger.debug("Closing connection %r", connection)
317+
def _close_connection(
318+
self, connection: DBAPIConnection, *, terminate: bool = False
319+
) -> None:
320+
self.logger.debug(
321+
"%s connection %r",
322+
"Hard-closing" if terminate else "Closing",
323+
connection,
324+
)
315325
try:
316-
self._dialect.do_close(connection)
326+
if terminate:
327+
self._dialect.do_terminate(connection)
328+
else:
329+
self._dialect.do_close(connection)
317330
except Exception:
318331
self.logger.error(
319332
"Exception closing connection %r", connection, exc_info=True
@@ -742,7 +755,7 @@ def invalidate(
742755
if soft:
743756
self._soft_invalidate_time = time.time()
744757
else:
745-
self.__close()
758+
self.__close(terminate=True)
746759
self.dbapi_connection = None
747760

748761
def get_connection(self) -> DBAPIConnection:
@@ -789,7 +802,7 @@ def get_connection(self) -> DBAPIConnection:
789802
recycle = True
790803

791804
if recycle:
792-
self.__close()
805+
self.__close(terminate=True)
793806
self.info.clear() # type: ignore # our info is always present
794807

795808
self.__connect()
@@ -804,12 +817,14 @@ def _is_hard_or_soft_invalidated(self) -> bool:
804817
or (self._soft_invalidate_time > self.starttime)
805818
)
806819

807-
def __close(self) -> None:
820+
def __close(self, *, terminate: bool = False) -> None:
808821
self.finalize_callback.clear()
809822
if self.__pool.dispatch.close:
810823
self.__pool.dispatch.close(self.dbapi_connection, self)
811824
assert self.dbapi_connection is not None
812-
self.__pool._close_connection(self.dbapi_connection)
825+
self.__pool._close_connection(
826+
self.dbapi_connection, terminate=terminate
827+
)
813828
self.dbapi_connection = None
814829

815830
def __connect(self) -> None:
@@ -877,7 +892,9 @@ def _finalize_fairy(
877892
dbapi_connection = connection_record.dbapi_connection
878893

879894
# null pool is not _is_asyncio but can be used also with async dialects
880-
dont_restore_gced = pool._dialect.is_async
895+
dont_restore_gced = (
896+
pool._dialect.is_async and not pool._dialect.has_terminate
897+
)
881898

882899
if dont_restore_gced:
883900
detach = connection_record is None or is_gc_cleanup
@@ -923,8 +940,9 @@ def _finalize_fairy(
923940
message = (
924941
"The garbage collector is trying to clean up "
925942
f"connection {dbapi_connection!r}. This feature is "
926-
"unsupported on async "
927-
"dbapi, since no IO can be performed at this stage to "
943+
"unsupported on asyncio "
944+
'dbapis that lack a "terminate" feature, since no '
945+
"IO can be performed at this stage to "
928946
"reset the connection. Please close out all "
929947
"connections when they are no longer used, calling "
930948
"``close()`` or using a context manager to "

‎test/engine/test_logging.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -453,7 +453,7 @@ def _test_queuepool(self, q, dispose=True):
453453
"Connection %r checked out from pool",
454454
"Connection %r being returned to pool%s",
455455
"Connection %s rollback-on-return",
456-
"Closing connection %r",
456+
"%s connection %r",
457457
]
458458
+ (["Pool disposed. %s"] if dispose else []),
459459
)

‎test/engine/test_pool.py‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -87,10 +87,13 @@ def _queuepool_fixture(self, **kw):
8787
def _queuepool_dbapi_fixture(self, **kw):
8888
dbapi = MockDBAPI()
8989
_is_asyncio = kw.pop("_is_asyncio", False)
90+
_has_terminate = kw.pop("_has_terminate", False)
9091
p = pool.QueuePool(creator=lambda: dbapi.connect("foo.db"), **kw)
9192
if _is_asyncio:
9293
p._is_asyncio = True
9394
p._dialect = _AsyncConnDialect()
95+
if _has_terminate:
96+
p._dialect.has_terminate = True
9497
return dbapi, p
9598

9699

@@ -445,8 +448,10 @@ def checkout(*arg, **kw):
445448

446449
return p, canary
447450

448-
def _checkin_event_fixture(self, _is_asyncio=False):
449-
p = self._queuepool_fixture(_is_asyncio=_is_asyncio)
451+
def _checkin_event_fixture(self, _is_asyncio=False, _has_terminate=False):
452+
p = self._queuepool_fixture(
453+
_is_asyncio=_is_asyncio, _has_terminate=_has_terminate
454+
)
450455
canary = []
451456

452457
@event.listens_for(p, "checkin")
@@ -721,9 +726,12 @@ def test_invalidate_event_exception(self):
721726
assert canary.call_args_list[0][0][0] is dbapi_con
722727
assert canary.call_args_list[0][0][2] is exc
723728

724-
@testing.combinations((True,), (False,))
725-
def test_checkin_event_gc(self, detach_gced):
726-
p, canary = self._checkin_event_fixture(_is_asyncio=detach_gced)
729+
@testing.combinations((True,), (False,), argnames="is_asyncio")
730+
@testing.combinations((True,), (False,), argnames="has_terminate")
731+
def test_checkin_event_gc(self, is_asyncio, has_terminate):
732+
p, canary = self._checkin_event_fixture(
733+
_is_asyncio=is_asyncio, _has_terminate=has_terminate
734+
)
727735

728736
c1 = p.connect()
729737

@@ -733,6 +741,8 @@ def test_checkin_event_gc(self, detach_gced):
733741
del c1
734742
lazy_gc()
735743

744+
detach_gced = is_asyncio and not has_terminate
745+
736746
if detach_gced:
737747
# "close_detached" is not called because for asyncio the
738748
# connection is just lost.

0 commit comments

Comments
 (0)