Skip to content

Commit cdc4118

Browse files
authored
Merge branch 'main' into fix/worker-lost-after-success-stored
2 parents 072ea29 + 451381d commit cdc4118

20 files changed

Lines changed: 585 additions & 98 deletions

File tree

‎celery/app/task.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1091,12 +1091,13 @@ def replace(self, sig):
10911091
sig.link(callback)
10921092
for errback in maybe_list(self.request.errbacks) or []:
10931093
sig.link_error(errback)
1094-
# If the replacement signature is a chain, we need to push callbacks
1094+
# If the replacement signature is a chain, we need to move callbacks
10951095
# down to the final task so they run at the right time even if we
1096-
# proceed to link further tasks from the original request below
1096+
# proceed to link further tasks from the original request below.
1097+
# Leaving them on the chain as well would link them twice.
10971098
if isinstance(sig, _chain) and "link" in sig.options:
10981099
final_task_links = sig.tasks[-1].options.setdefault("link", [])
1099-
final_task_links.extend(maybe_list(sig.options["link"]))
1100+
final_task_links.extend(maybe_list(sig.options.pop("link")))
11001101
# We need to freeze the replacement signature with the current task's
11011102
# ID to ensure that we don't disassociate it from the existing task IDs
11021103
# which would break previously constructed results objects.

‎celery/app/utils.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,9 @@ def without_defaults(self):
146146
return Settings({}, self.maps[:-1])
147147

148148
def value_set_for(self, key):
149-
return key in self.without_defaults()
149+
self.finalize()
150+
# Also find settings given with a namespace or an old setting name.
151+
return key in Settings({}, self.maps[:-1], self._keys, self.prefix)
150152

151153
def find_option(self, name, namespace=''):
152154
"""Search for option by name.

‎celery/backends/database/__init__.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,12 @@ def __init__(self, dburi=None, engine_options=None, url=None, **kwargs):
6868
# Override retry defaults to preserve backward compatibility.
6969
# Previously, DatabaseBackend used a custom @retry decorator that always
7070
# retried with max_retries=3. We maintain this behavior by default.
71-
self.always_retry = conf.get('result_backend_always_retry', True)
72-
self.max_retries = conf.get('result_backend_max_retries', 3)
71+
# Both settings always have a value, so check if the user set them.
72+
# None also means the default.
73+
if not conf.value_set_for('result_backend_always_retry') or self.always_retry is None:
74+
self.always_retry = True
75+
if not conf.value_set_for('result_backend_max_retries') or self.max_retries is None:
76+
self.max_retries = 3
7377

7478
if self.extended_result:
7579
self.task_cls = TaskExtended

‎celery/concurrency/asynpool.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -624,7 +624,12 @@ def register_with_event_loop(self, hub):
624624
def _create_timelimit_handlers(self, hub):
625625
"""Create handlers used to implement time limits."""
626626
call_later = hub.call_later
627-
trefs = self._tref_for_id = WeakValueDictionary()
627+
# Keep the existing mapping when registering again after a broker
628+
# reconnect, so the time limits of tasks that are still running
629+
# can still be cancelled when they finish.
630+
trefs = getattr(self, '_tref_for_id', None)
631+
if trefs is None:
632+
trefs = self._tref_for_id = WeakValueDictionary()
628633

629634
def on_timeout_set(R, soft, hard):
630635
if soft:

‎celery/schedules.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@
1414
from . import current_app
1515
from .exceptions import ImproperlyConfigured
1616
from .utils.collections import AttributeDict
17-
from .utils.time import (ffwd, humanize_seconds, localize, maybe_make_aware, maybe_timedelta, remaining, timezone,
18-
weekday, yearmonth)
17+
from .utils.time import (ffwd, humanize_seconds, localize, maybe_make_aware, maybe_timedelta, remaining,
18+
subtract_datetimes, timezone, weekday, yearmonth)
1919

2020
__all__ = (
2121
'ParseException', 'schedule', 'crontab', 'crontab_parser',
@@ -907,8 +907,7 @@ def remaining_estimate(self, last_run_at: datetime) -> timedelta:
907907
next_utc = self._next_event(self.ephem.Date(last_run_at_utc))
908908
next = self.maybe_make_aware(next_utc.datetime())
909909
now = self.maybe_make_aware(self.now())
910-
delta = next - now
911-
return delta
910+
return subtract_datetimes(next, now)
912911

913912
def is_due(self, last_run_at: datetime) -> tuple[bool, datetime]:
914913
"""Return tuple of ``(is_due, next_time_to_run)``.

‎celery/utils/collections.py‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -630,23 +630,30 @@ def purge(self, now=None):
630630
# time based expiring:
631631
if self.expires:
632632
while len(self._data) > self.minlen >= 0:
633-
inserted_time = self._heap[0][0]
634-
if inserted_time + self.expires > now:
633+
entry = self._heap[0]
634+
if not self._is_current(entry):
635+
# left behind by discard(), or by re-adding the item
636+
heappop(self._heap)
637+
continue
638+
if entry[0] + self.expires > now:
635639
break # oldest item hasn't expired yet
636640
self.pop()
637641

638642
def pop(self, default: Any = None) -> Any:
639643
"""Remove and return the oldest item, or :const:`None` when empty."""
640644
while self._heap:
641-
_, _, item = heappop(self._heap)
642-
try:
643-
self._data.pop(item)
644-
except KeyError:
645-
pass
646-
else:
647-
return item
645+
entry = heappop(self._heap)
646+
if self._is_current(entry):
647+
del self._data[entry[2]]
648+
return entry[2]
648649
return default
649650

651+
def _is_current(self, entry):
652+
# type: (Tuple[float, int, Any]) -> bool
653+
# discard() doesn't remove an item's heap entry, so the heap can
654+
# still hold an outdated entry for an item that was re-added since.
655+
return self._data.get(entry[2]) is entry
656+
650657
def as_dict(self):
651658
# type: () -> Dict
652659
"""Whole set as serializable dictionary.

‎celery/worker/consumer/consumer.py‎

Lines changed: 43 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,14 @@ class Consumer:
159159
#: as sending heartbeats.
160160
timer = None
161161

162+
#: Entries on the event loop timer before the consumer last started
163+
#: connecting to the broker.
164+
timer_entries_before_start = None
165+
166+
#: Entries the event loop registered on its timer for the current
167+
#: connection, or :const:`None` if it has not registered them yet.
168+
timer_entries_registered = None
169+
162170
restart_count = -1 # first start is the same as a restart
163171

164172
#: This flag will be turned off after the first failed
@@ -387,6 +395,13 @@ def start(self):
387395
recoverable_errors = (self.connection_errors + self.channel_errors)
388396
else:
389397
recoverable_errors = self.connection_errors
398+
if self.hub is not None:
399+
# Entries added to the timer from here on, by connecting
400+
# and starting the event loop, belong to this connection
401+
# (see :func:`celery.worker.loops.asynloop`).
402+
self.timer_entries_before_start = loops._timer_entries(
403+
self.hub.timer)
404+
self.timer_entries_registered = None
390405
try:
391406
blueprint.start(self)
392407
except recoverable_errors as exc:
@@ -413,8 +428,31 @@ def start(self):
413428
else:
414429
self.on_connection_error_before_connected(exc)
415430
self.on_close()
431+
self._cancel_connection_timer_entries()
416432
blueprint.restart(self)
417433

434+
def _cancel_connection_timer_entries(self):
435+
"""Cancel the timer entries registered for the failed connection.
436+
437+
:func:`~celery.worker.loops.asynloop` records the entries it
438+
registered for the connection. If the connection failed before
439+
that, e.g. in a bootstep that runs after the transport connected,
440+
every entry added since the consumer started connecting belongs to
441+
it: the event loop was not running, so nothing else added entries
442+
in the meantime. Without this, those entries would be part of the
443+
next attempt's snapshot and never be cancelled.
444+
445+
Cancelling an entry twice is harmless, so this also runs after
446+
:func:`~celery.worker.loops.asynloop` cancelled its own entries.
447+
"""
448+
if self.hub is None or self.timer_entries_before_start is None:
449+
return
450+
entries = self.timer_entries_registered
451+
if entries is None:
452+
entries = (loops._timer_entries(self.hub.timer) -
453+
self.timer_entries_before_start)
454+
loops._cancel_timer_entries(self.hub.timer, entries)
455+
418456
def _get_connection_retry_type(self, is_connection_loss_on_startup):
419457
return ('broker_connection_retry_on_startup'
420458
if (is_connection_loss_on_startup
@@ -555,10 +593,11 @@ def on_close(self):
555593
# entries (through self.timer.cancel(), not entry.cancel()
556594
# directly, since e.g. the Eventlet timer relies on that to catch
557595
# GreenletExit) so the callback can't fire after we've torn down
558-
# this connection (the synloop error path doesn't clear the timer
559-
# the way asynloop's hub.reset()/hub.timer.clear() does), then drop
560-
# our copies since the broker may redeliver these tasks to another
561-
# worker on reconnect.
596+
# this connection (neither event loop cancels them: synloop's error
597+
# path cancels nothing, and asynloop's only cancels the entries it
598+
# registered for the connection, which doesn't include these), then
599+
# drop our copies since the broker may redeliver these tasks to
600+
# another worker on reconnect.
562601
for r in tuple(scheduled_requests):
563602
entry = r._eta_timer_entry
564603
if entry is not None and self.timer is not None:

‎celery/worker/loops.py‎

Lines changed: 40 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,21 @@ def tick(rate):
4747
return heartbeat_error
4848

4949

50+
def _timer_entries(timer):
51+
return {scheduled.entry for scheduled in timer.queue}
52+
53+
54+
def _cancel_timer_entries(timer, entries):
55+
# Cancel each entry on its own, so one that fails to cancel does not
56+
# leave the remaining entries of the connection live.
57+
for entry in entries:
58+
try:
59+
timer.cancel(entry)
60+
except Exception as exc: # pylint: disable=broad-except
61+
logger.exception(
62+
'Error cancelling hub timer entry %r: %r', entry, exc)
63+
64+
5065
def asynloop(obj, connection, consumer, blueprint, hub, qos,
5166
heartbeat, clock, hbrate=2.0):
5267
"""Non-blocking event loop."""
@@ -56,6 +71,13 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos,
5671

5772
on_task_received = obj.create_task_handler()
5873

74+
# The consumer records the timer entries that existed before it started
75+
# connecting, so that the entries registered by the transport when
76+
# connecting are cancelled along with the ones registered below.
77+
timer_entries_before = obj.timer_entries_before_start
78+
if timer_entries_before is None:
79+
timer_entries_before = _timer_entries(hub.timer)
80+
5981
heartbeat_error = _enable_amqheartbeats(hub.timer, connection, rate=hbrate)
6082
dispatcher_heartbeat_error = [None]
6183
if obj.event_dispatcher:
@@ -65,6 +87,11 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos,
6587
consumer.on_message = on_task_received
6688
obj.controller.register_with_event_loop(hub)
6789
obj.register_with_event_loop(hub)
90+
registered_timer_entries = (
91+
_timer_entries(hub.timer) - timer_entries_before)
92+
# The consumer cancels these if the connection fails before the loop
93+
# below is running (see :meth:`Consumer.start`).
94+
obj.timer_entries_registered = registered_timer_entries
6895
consumer.consume()
6996
obj.on_ready()
7097

@@ -106,12 +133,11 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos,
106133
except Exception:
107134
# Reset the hub on error (e.g. connection loss) to clean up
108135
# stale file descriptors and callbacks from the old connection.
109-
# Also clear the timer queue so that stale periodic entries added by
110-
# register_with_event_loop (e.g. maybe_restore_messages) do not fire
111-
# against the broken connection after reconnect and trigger another
112-
# crash before the new connection is fully established.
113-
# All hub timers are re-registered during blueprint.start() once this
114-
# exception propagates and the consumer reconnects.
136+
# Also cancel the timer entries registered while connecting and
137+
# starting this loop, so that stale periodic entries (e.g.
138+
# maybe_restore_messages) do not fire against the broken connection
139+
# after reconnect and trigger another crash before the new
140+
# connection is fully established.
115141
# We intentionally do NOT reset on normal exit (graceful shutdown)
116142
# so that timers (e.g. heartbeat) keep firing while the pool drains.
117143
# WorkerShutdown/WorkerTerminate extend SystemExit (not Exception)
@@ -121,18 +147,14 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos,
121147
except Exception as exc: # pylint: disable=broad-except
122148
logger.exception(
123149
'Error cleaning up after event loop: %r', exc)
124-
# Clear stale timer entries accumulated across reconnects (e.g.
125-
# maybe_restore_messages registered via call_repeatedly). Without
126-
# this, each reconnect appends a new entry; all of them fire during
127-
# the reconnect window, raise again, and trigger another restart.
128-
# Use a separate try/except so this always runs even if hub.reset()
129-
# raised above. Timers are re-registered by register_with_event_loop
130-
# when blueprint.start() is called after reconnect.
131-
try:
132-
hub.timer.clear()
133-
except Exception as exc: # pylint: disable=broad-except
134-
logger.exception(
135-
'Error clearing hub timer after event loop: %r', exc)
150+
# These entries are registered again when the consumer reconnects
151+
# and calls asynloop again; without cancelling them, each reconnect
152+
# adds another copy. The timer must not be cleared: it is shared
153+
# with the rest of the worker, and other entries on it, such as the
154+
# soft/hard time limits of tasks that keep running across the
155+
# reconnect, are not registered again (#10888).
156+
# This runs even if hub.reset() raised above.
157+
_cancel_timer_entries(hub.timer, registered_timer_entries)
136158
raise
137159

138160

‎docs/userguide/configuration.rst‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -902,7 +902,7 @@ Default: :const:`False`
902902
setting to :const:`True` by default to preserve backward compatibility
903903
with the automatic retry behavior that was previously provided by an
904904
internal ``@retry`` decorator. Other backends continue to default to
905-
:const:`False`.
905+
:const:`False`. Setting it to :const:`None` also gives :const:`True`.
906906

907907
If enabled, the backend will try to retry on the event of recoverable
908908
exceptions instead of propagating the exception.
@@ -942,6 +942,7 @@ Default: Inf
942942
setting to ``3`` by default to preserve backward compatibility with the
943943
behavior previously provided by an internal ``@retry`` decorator.
944944
Other backends continue to default to :const:`Inf` (unlimited retries).
945+
Setting it to :const:`None` also gives ``3``.
945946

946947
This is the maximum number of retries in case of recoverable exceptions.
947948

‎t/integration/tasks.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,12 @@ def record_replaced_task_nesting(self, replacements, nesting=None):
168168
return nesting
169169

170170

171+
@shared_task(bind=True)
172+
def add_replaced_with_chain(self, x, y):
173+
"""Add two numbers (via a chain of the add and identity tasks)."""
174+
raise self.replace(chain(add.s(x, y), identity.s()))
175+
176+
171177
@shared_task(bind=True)
172178
def replace_with_chain(self, *args, link_msg=None):
173179
c = chain(identity.s(*args), identity.s())

0 commit comments

Comments
 (0)