Skip to content

Commit 67a1cbc

Browse files
Make pool min_size a maintained connection floor (MagicStack#1323)
Currently `min_size` only controls how many connections a pool opens at startup. Once idle connections expire through `max_inactive_connection_lifetime`, or are closed by the server, the pool can drain to zero regardless of `min_size`. Split the old meaning into two parameters: * `init_size`: number of connections opened when the pool is created. Defaults to `min_size`, so existing callers see the same startup behaviour. * `min_size`: number of live connections the pool attempts to maintain at all times. Changes in behavior: * The inactivity callback no longer closes a connection when the pool is at or below `min_size`. It stays open, and its timer is re-armed on the next acquire/release. * When a pooled connection closes, including an idle one that was not checked out, a background maintenance task reconnects holders until the floor is restored. Failed reconnects are logged and retried with exponential backoff capped at 60 seconds. * The holder queue prefers holders that already have a live connection (LIFO within each group), and maintenance takes only disconnected holders. * A pool whose initialization fails terminates any connections it already opened. * A holder that finishes connecting after the pool has started closing closes the new connection instead of keeping it. Note: to keep the old behaviour with no effective floor, pass `init_size=N`, `min_size=0` where you previously passed `min_size=N`. Co-authored-by: Elvis Pranskevichus <[email protected]>
1 parent 25aaaf5 commit 67a1cbc

8 files changed

Lines changed: 667 additions & 75 deletions

File tree

‎asyncpg/_testbase/__init__.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,7 @@ def _shutdown_cluster(cluster):
267267

268268

269269
def create_pool(dsn=None, *,
270+
init_size=None,
270271
min_size=10,
271272
max_size=10,
272273
max_queries=50000,
@@ -281,6 +282,7 @@ def create_pool(dsn=None, *,
281282
**connect_kwargs):
282283
return pool_class(
283284
dsn,
285+
init_size=init_size,
284286
min_size=min_size,
285287
max_size=max_size,
286288
max_queries=max_queries,
@@ -366,10 +368,16 @@ def setUp(self):
366368
self._pools = []
367369

368370
def tearDown(self):
369-
super().tearDown()
371+
maintenance_tasks = []
370372
for pool in self._pools:
371373
pool.terminate()
374+
if pool._maintenance_task is not None:
375+
maintenance_tasks.append(pool._maintenance_task)
376+
if maintenance_tasks:
377+
self.loop.run_until_complete(asyncio.gather(
378+
*maintenance_tasks, return_exceptions=True))
372379
self._pools = []
380+
super().tearDown()
373381

374382
def create_pool(self, pool_class=pg_pool.Pool,
375383
connection_class=pg_connection.Connection, **kwargs):

‎asyncpg/connect_utils.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1158,7 +1158,16 @@ async def __connect_addr(
11581158
else:
11591159
connector = loop.create_connection(proto_factory, *addr)
11601160

1161-
tr, pr = await connector
1161+
try:
1162+
tr, pr = await connector
1163+
except (Exception, asyncio.CancelledError):
1164+
# The protocol can exist before create_connection() returns. If
1165+
# that operation is cancelled, nobody will await authentication.
1166+
if not connected.done():
1167+
connected.cancel()
1168+
elif not connected.cancelled():
1169+
connected.exception()
1170+
raise
11621171

11631172
try:
11641173
await connected

‎asyncpg/connection.py‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ class Connection(metaclass=ConnectionMeta):
5454
'_intro_query', '_reset_query', '_proxy',
5555
'_stmt_exclusive_section', '_config', '_params', '_addr',
5656
'_log_listeners', '_termination_listeners', '_cancellations',
57+
'_pool_holder',
5758
'_source_traceback', '_query_loggers', '__weakref__')
5859

5960
def __init__(self, protocol, transport, loop,
@@ -105,6 +106,7 @@ def __init__(self, protocol, transport, loop,
105106

106107
self._reset_query = None
107108
self._proxy = None
109+
self._pool_holder = None
108110

109111
# Used to serialize operations that might involve anonymous
110112
# statements. Specifically, we want to make the following
@@ -134,6 +136,10 @@ def __del__(self):
134136

135137
warnings.warn(msg, ResourceWarning)
136138
if not self._loop.is_closed():
139+
# A weak holder reference may still be live during GC.
140+
# Finalization must not notify the pool and restart
141+
# maintenance.
142+
self._pool_holder = None
137143
self.terminate()
138144

139145
async def add_listener(self, channel, callback):
@@ -1599,10 +1605,11 @@ def _cleanup(self):
15991605
# Free the resources associated with this connection.
16001606
# This must be called when a connection is terminated.
16011607

1602-
if self._proxy is not None:
1603-
# Connection is a member of a pool, so let the pool
1604-
# know that this connection is dead.
1605-
self._proxy._holder._release_on_close()
1608+
if self._pool_holder is not None:
1609+
# Idle connections have no proxy, but still belong to a holder.
1610+
holder, self._pool_holder = self._pool_holder(), None
1611+
if holder is not None and holder._con is self:
1612+
holder._release_on_close()
16061613

16071614
self._mark_stmts_as_closed()
16081615
self._listeners.clear()

0 commit comments

Comments
 (0)