@@ -47,6 +47,21 @@ def tick(rate):
4747 return heartbeat_error
4848
4949
50+ def _timer_entries (timer ):
51+ return {scheduled .entry for scheduled in timer .queue }
52+
53+
54+ def _cancel_timer_entries (timer , entries ):
55+ # Cancel each entry on its own, so one that fails to cancel does not
56+ # leave the remaining entries of the connection live.
57+ for entry in entries :
58+ try :
59+ timer .cancel (entry )
60+ except Exception as exc : # pylint: disable=broad-except
61+ logger .exception (
62+ 'Error cancelling hub timer entry %r: %r' , entry , exc )
63+
64+
5065def asynloop (obj , connection , consumer , blueprint , hub , qos ,
5166 heartbeat , clock , hbrate = 2.0 ):
5267 """Non-blocking event loop."""
@@ -56,6 +71,13 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos,
5671
5772 on_task_received = obj .create_task_handler ()
5873
74+ # The consumer records the timer entries that existed before it started
75+ # connecting, so that the entries registered by the transport when
76+ # connecting are cancelled along with the ones registered below.
77+ timer_entries_before = obj .timer_entries_before_start
78+ if timer_entries_before is None :
79+ timer_entries_before = _timer_entries (hub .timer )
80+
5981 heartbeat_error = _enable_amqheartbeats (hub .timer , connection , rate = hbrate )
6082 dispatcher_heartbeat_error = [None ]
6183 if obj .event_dispatcher :
@@ -65,6 +87,11 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos,
6587 consumer .on_message = on_task_received
6688 obj .controller .register_with_event_loop (hub )
6789 obj .register_with_event_loop (hub )
90+ registered_timer_entries = (
91+ _timer_entries (hub .timer ) - timer_entries_before )
92+ # The consumer cancels these if the connection fails before the loop
93+ # below is running (see :meth:`Consumer.start`).
94+ obj .timer_entries_registered = registered_timer_entries
6895 consumer .consume ()
6996 obj .on_ready ()
7097
@@ -106,12 +133,11 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos,
106133 except Exception :
107134 # Reset the hub on error (e.g. connection loss) to clean up
108135 # stale file descriptors and callbacks from the old connection.
109- # Also clear the timer queue so that stale periodic entries added by
110- # register_with_event_loop (e.g. maybe_restore_messages) do not fire
111- # against the broken connection after reconnect and trigger another
112- # crash before the new connection is fully established.
113- # All hub timers are re-registered during blueprint.start() once this
114- # exception propagates and the consumer reconnects.
136+ # Also cancel the timer entries registered while connecting and
137+ # starting this loop, so that stale periodic entries (e.g.
138+ # maybe_restore_messages) do not fire against the broken connection
139+ # after reconnect and trigger another crash before the new
140+ # connection is fully established.
115141 # We intentionally do NOT reset on normal exit (graceful shutdown)
116142 # so that timers (e.g. heartbeat) keep firing while the pool drains.
117143 # WorkerShutdown/WorkerTerminate extend SystemExit (not Exception)
@@ -121,18 +147,14 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos,
121147 except Exception as exc : # pylint: disable=broad-except
122148 logger .exception (
123149 'Error cleaning up after event loop: %r' , exc )
124- # Clear stale timer entries accumulated across reconnects (e.g.
125- # maybe_restore_messages registered via call_repeatedly). Without
126- # this, each reconnect appends a new entry; all of them fire during
127- # the reconnect window, raise again, and trigger another restart.
128- # Use a separate try/except so this always runs even if hub.reset()
129- # raised above. Timers are re-registered by register_with_event_loop
130- # when blueprint.start() is called after reconnect.
131- try :
132- hub .timer .clear ()
133- except Exception as exc : # pylint: disable=broad-except
134- logger .exception (
135- 'Error clearing hub timer after event loop: %r' , exc )
150+ # These entries are registered again when the consumer reconnects
151+ # and calls asynloop again; without cancelling them, each reconnect
152+ # adds another copy. The timer must not be cleared: it is shared
153+ # with the rest of the worker, and other entries on it, such as the
154+ # soft/hard time limits of tasks that keep running across the
155+ # reconnect, are not registered again (#10888).
156+ # This runs even if hub.reset() raised above.
157+ _cancel_timer_entries (hub .timer , registered_timer_entries )
136158 raise
137159
138160
0 commit comments