Skip to content

Commit 5a635c2

Browse files
authored
Merge branch 'main' into fix/canvas-repr-remove-repeating
2 parents 886aba1 + 6cb6ff9 commit 5a635c2

5 files changed

Lines changed: 400 additions & 14 deletions

File tree

‎celery/backends/mongodb.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
from celery import states
99
from celery.exceptions import ImproperlyConfigured
10+
from celery.utils.time import add_seconds_to_datetime
1011

1112
from .base import BaseBackend
1213

@@ -283,11 +284,12 @@ def cleanup(self):
283284
if not self.expires:
284285
return
285286

287+
cutoff = add_seconds_to_datetime(self.app.now(), -self.expires_delta.total_seconds())
286288
self.collection.delete_many(
287-
{'date_done': {'$lt': self.app.now() - self.expires_delta}},
289+
{'date_done': {'$lt': cutoff}},
288290
)
289291
self.group_collection.delete_many(
290-
{'date_done': {'$lt': self.app.now() - self.expires_delta}},
292+
{'date_done': {'$lt': cutoff}},
291293
)
292294

293295
def __reduce__(self, args=(), kwargs=None):

‎celery/canvas.py‎

Lines changed: 57 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1091,6 +1091,29 @@ def unchain_tasks(self):
10911091
task.link_error(sig)
10921092
return tasks
10931093

1094+
def _unroll_chords(self, app=None):
1095+
"""Split nested chord bodies before freezing.
1096+
1097+
This mutates the chain in place; call it only on an execution copy.
1098+
"""
1099+
app = app or self.app
1100+
if self._use_link or (self._use_link is None and app.conf.task_protocol == 1):
1101+
# A failed header needs the whole body to reach the final result
1102+
# when callbacks, rather than the chain message field, link tasks.
1103+
return
1104+
pending = deque(self.tasks)
1105+
tasks = []
1106+
while pending:
1107+
task = pending.popleft()
1108+
if isinstance(task, _chain):
1109+
task = task.clone()
1110+
task._unroll_chords(app)
1111+
if isinstance(task, _chord) and _is_chain_led_by_task(task.body):
1112+
pending.extendleft(reversed(task._unroll_body()))
1113+
else:
1114+
tasks.append(task)
1115+
self.tasks = tasks
1116+
10941117
def apply_async(self, args=None, kwargs=None, **options):
10951118
# python is best at unpacking kwargs, so .run is here to do that.
10961119
args = args if args else ()
@@ -1284,15 +1307,9 @@ def prepare_steps(self, args, kwargs, tasks,
12841307
continue
12851308

12861309
if clone and isinstance(task, chord) and _is_chain_led_by_task(task.body):
1287-
# chord(header, chain(a, b, c)) -> chord(header, a), b, c
1288-
# Every header task carries a copy of the body, so a body
1289-
# holding the rest of the chain is copied once per header
1290-
# task, at every chord. Freezing (clone=False) assigns ids
1291-
# to the tasks in place and keeps the chain as written.
1292-
task = task.clone()
1293-
body_tasks = task.body.unchain_tasks()
1294-
task.body = body_tasks[0].clone(**_first_task_options(task.body))
1295-
steps_extend([task, *body_tasks[1:]])
1310+
# Freezing (clone=False) assigns ids to the tasks in place
1311+
# and keeps the chain as written.
1312+
steps_extend(task._unroll_body())
12961313
continue
12971314

12981315
# first task gets partial args from chain
@@ -1762,7 +1779,11 @@ def apply(self, args=None, kwargs=None, **options):
17621779
if not self.tasks:
17631780
return self.freeze() # empty group returns GroupResult
17641781
options, group_id, root_id = self._freeze_gid(options)
1765-
tasks = self._prepared(self.tasks, [], group_id, root_id, app)
1782+
# Freezing a chain that ends in a group builds a GroupResult that
1783+
# subscribes to the result backend, which eager apply never uses.
1784+
tasks = self._prepared(
1785+
self.tasks, [], group_id, root_id, app, freeze=False,
1786+
)
17661787
return app.GroupResult(group_id, [
17671788
sig.apply(args=args, kwargs=kwargs, **options) for sig, _, _ in tasks
17681789
])
@@ -1820,7 +1841,7 @@ def link_error(self, sig):
18201841
def _prepared(self, tasks, partial_args, group_id, root_id, app,
18211842
CallableSignature=abstract.CallableSignature,
18221843
from_dict=Signature.from_dict,
1823-
isinstance=isinstance, tuple=tuple):
1844+
isinstance=isinstance, tuple=tuple, *, freeze=True):
18241845
"""Recursively unroll the group into a generator of its tasks.
18251846
18261847
This is used by :meth:`apply_async` and :meth:`apply` to
@@ -1842,6 +1863,8 @@ def _prepared(self, tasks, partial_args, group_id, root_id, app,
18421863
isinstance (fun): Function to check if an object is an instance
18431864
of a class.
18441865
tuple (class): A tuple-like class.
1866+
freeze (bool): Freeze each task to get its result. When false,
1867+
the generator yields ``None`` in place of the result.
18451868
18461869
Returns:
18471870
generator: A generator for the unrolled group tasks.
@@ -1859,6 +1882,7 @@ def _prepared(self, tasks, partial_args, group_id, root_id, app,
18591882
# needs yield_from :(
18601883
unroll = task._prepared(
18611884
task.tasks, partial_args, group_id, root_id, app,
1885+
freeze=freeze,
18621886
)
18631887
yield from unroll
18641888
elif isinstance(task, _chain) and not task.tasks:
@@ -1869,7 +1893,10 @@ def _prepared(self, tasks, partial_args, group_id, root_id, app,
18691893
else:
18701894
if partial_args and not task.immutable:
18711895
task.args = tuple(partial_args) + tuple(task.args)
1872-
yield task, task.freeze(group_id=group_id, root_id=root_id, group_index=index), group_id
1896+
result = task.freeze(
1897+
group_id=group_id, root_id=root_id, group_index=index,
1898+
) if freeze else None
1899+
yield task, result, group_id
18731900

18741901
def _apply_tasks(self, tasks, producer=None, app=None, p=None,
18751902
add_to_parent=None, chord=None,
@@ -2400,6 +2427,11 @@ def run(self, header, body, partial_args, app=None, interval=None,
24002427
options.pop('task_id', None)
24012428
body.options.update(self._body_options(options))
24022429

2430+
# Freezing a nested chain first would embed its whole remaining tail
2431+
# in every inner chord header, before prepare_steps can split it.
2432+
# apply_async has already cloned the body; public freeze stays in place.
2433+
if isinstance(body, _chain):
2434+
body._unroll_chords(app)
24032435
body_task_id = task_id or uuid()
24042436
bodyres = body.freeze(body_task_id, group_id=group_id, root_id=root_id)
24052437

@@ -2435,6 +2467,19 @@ def run(self, header, body, partial_args, app=None, interval=None,
24352467
bodyres.parent = header_result
24362468
return bodyres
24372469

2470+
def _unroll_body(self):
2471+
"""Return this chord followed by the tail of its task-led chain body.
2472+
2473+
chord(header, chain(a, b, c)) -> chord(header, a), b, c
2474+
2475+
Every header task carries a copy of the body, so a body holding the
2476+
rest of the chain would be copied once per header task, at every chord.
2477+
"""
2478+
task = self.clone()
2479+
body_tasks = task.body.unchain_tasks()
2480+
task.body = body_tasks[0].clone(**_first_task_options(task.body))
2481+
return [task, *body_tasks[1:]]
2482+
24382483
def clone(self, *args, **kwargs):
24392484
signature = super().clone(*args, **kwargs)
24402485
# need to make copy of body

‎t/integration/test_canvas.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import collections
2+
import json
23
import re
34
import tempfile
45
import uuid
@@ -3627,6 +3628,56 @@ def test_chord_header_with_empty_chains(self, manager):
36273628
res = chord([chain(), add.s(1, 1), chain(), add.s(2, 2)], body=tsum.s())()
36283629
assert res.get(timeout=TIMEOUT) == 6
36293630

3631+
@pytest.mark.parametrize('serialized', [False, True])
3632+
@pytest.mark.parametrize('frozen', [False, True])
3633+
@flaky
3634+
def test_chain_of_implicit_chords_as_body(self, manager, serialized, frozen):
3635+
try:
3636+
manager.app.backend.ensure_chords_allowed()
3637+
except NotImplementedError as e:
3638+
raise pytest.skip(e.args[0])
3639+
3640+
steps = [tsum.s()]
3641+
for _ in range(5):
3642+
steps.extend([group(add.s(1), add.s(2)), tsum.s()])
3643+
callback = identity.s()
3644+
callback_result = callback.freeze()
3645+
steps[-1].link(callback)
3646+
workflow = chord([add.si(1, 1), add.si(2, 2)], chain(*steps))
3647+
expected_id = str(uuid.uuid4())
3648+
if frozen:
3649+
assert workflow.freeze(expected_id).id == expected_id
3650+
if serialized:
3651+
workflow = signature(json.loads(json.dumps(workflow)))
3652+
3653+
result = workflow.apply_async(task_id=expected_id)
3654+
3655+
assert result.id == expected_id
3656+
# The outer header sums to 6; each inner group maps x to 2*x + 3.
3657+
assert result.get(timeout=TIMEOUT) == 285
3658+
assert callback_result.get(timeout=TIMEOUT) == 285
3659+
3660+
@pytest.mark.parametrize('failure', ['header', 'body'])
3661+
@flaky
3662+
def test_chain_of_implicit_chords_body_failure(self, manager, failure):
3663+
if manager.app.conf.task_protocol == 1:
3664+
raise pytest.skip('Requires the chain message field (task_protocol=2).')
3665+
if not manager.app.conf.result_backend.startswith('redis'):
3666+
raise pytest.skip('Requires redis result backend.')
3667+
3668+
redis_key = str(uuid.uuid4())
3669+
first_group = group(fail.si(), add.si(1, 2)) if failure == 'header' else group(add.si(1, 2))
3670+
callback = fail.si() if failure == 'body' else tsum.s()
3671+
inner_body = chain(callback, group(add.s(1), add.s(2)), tsum.s())
3672+
inner_body.link_error(redis_echo.si('failed', redis_key=redis_key))
3673+
workflow = chord([add.si(1, 1)], chain(tsum.s(), chord(first_group, inner_body)))
3674+
3675+
result = workflow.delay()
3676+
3677+
with pytest.raises((ExpectedException, ChordError)):
3678+
result.get(timeout=TIMEOUT)
3679+
await_redis_echo('failed', redis_key=redis_key)
3680+
36303681

36313682
class test_signature_serialization:
36323683
"""

‎t/unit/backends/test_mongodb.py‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
import dns.version
66
import pymongo
77
import pytest
8+
from bson import BSON
9+
from dateutil import tz as dateutil_tz
810
from kombu.exceptions import EncodeError
911

1012
try:
@@ -712,6 +714,171 @@ def now_func():
712714
self.backend.cleanup()
713715
mock_collection.delete_many.assert_not_called()
714716

717+
@pytest.mark.parametrize('tz_factory', [
718+
pytest.param(ZoneInfo, id='zoneinfo'),
719+
pytest.param(dateutil_tz.gettz, id='dateutil'),
720+
])
721+
@pytest.mark.parametrize('zone,now,expires,expected_cutoff', [
722+
pytest.param(
723+
'America/New_York', datetime.datetime(2026, 11, 1, 1, 30, fold=0), 30,
724+
datetime.datetime(2026, 11, 1, 5, 29, 30),
725+
id='fall-back-first-hour',
726+
),
727+
pytest.param(
728+
'America/New_York', datetime.datetime(2026, 11, 1, 1, 30, fold=1), 30,
729+
datetime.datetime(2026, 11, 1, 6, 29, 30),
730+
id='fall-back-repeated-hour',
731+
),
732+
pytest.param(
733+
'America/New_York', datetime.datetime(2026, 11, 1, 1, 30, microsecond=125000, fold=1), 30.125,
734+
datetime.datetime(2026, 11, 1, 6, 29, 30),
735+
id='fractional-expiration-during-fall-back',
736+
),
737+
pytest.param(
738+
'America/New_York', datetime.datetime(2026, 11, 1, 1, 30, microsecond=125000, fold=1),
739+
datetime.timedelta(seconds=30.125),
740+
datetime.datetime(2026, 11, 1, 6, 29, 30),
741+
id='timedelta-expiration-during-fall-back',
742+
),
743+
pytest.param(
744+
'America/New_York', datetime.datetime(2026, 11, 1, 2, 30), 2 * 60 * 60,
745+
datetime.datetime(2026, 11, 1, 5, 30),
746+
id='across-fall-back',
747+
),
748+
pytest.param(
749+
'America/New_York', datetime.datetime(2026, 3, 8, 3, 30), 2 * 60 * 60,
750+
datetime.datetime(2026, 3, 8, 5, 30),
751+
id='across-spring-forward',
752+
),
753+
pytest.param(
754+
'America/New_York', datetime.datetime(2026, 3, 9, 0, 30), 24 * 60 * 60,
755+
datetime.datetime(2026, 3, 8, 4, 30),
756+
id='elapsed-day-spring-forward',
757+
),
758+
pytest.param(
759+
'America/New_York', datetime.datetime(2026, 11, 2, 0, 30), 24 * 60 * 60,
760+
datetime.datetime(2026, 11, 1, 5, 30),
761+
id='elapsed-day-fall-back',
762+
),
763+
pytest.param(
764+
'Australia/Lord_Howe', datetime.datetime(2026, 4, 5, 1, 45, fold=1), 30,
765+
datetime.datetime(2026, 4, 4, 15, 14, 30),
766+
id='half-hour-fall-back',
767+
),
768+
pytest.param(
769+
'Australia/Lord_Howe', datetime.datetime(2026, 10, 4, 2, 45), 60 * 60,
770+
datetime.datetime(2026, 10, 3, 14, 45),
771+
id='half-hour-spring-forward',
772+
),
773+
pytest.param(
774+
'America/Havana', datetime.datetime(2026, 3, 8, 1, 30), 60 * 60,
775+
datetime.datetime(2026, 3, 8, 4, 30),
776+
id='midnight-spring-forward',
777+
),
778+
pytest.param(
779+
'America/Havana', datetime.datetime(2026, 11, 1, 0, 30, fold=1), 30,
780+
datetime.datetime(2026, 11, 1, 5, 29, 30),
781+
id='repeated-midnight',
782+
),
783+
pytest.param(
784+
'Asia/Kathmandu', datetime.datetime(1986, 1, 1, 0, 30), 60 * 60,
785+
datetime.datetime(1985, 12, 31, 17, 45),
786+
id='quarter-hour-midnight-gap',
787+
),
788+
pytest.param(
789+
'Pacific/Apia', datetime.datetime(2011, 12, 31, 0, 30), 24 * 60 * 60,
790+
datetime.datetime(2011, 12, 29, 10, 30),
791+
id='elapsed-day-skipped-date',
792+
),
793+
pytest.param(
794+
'America/New_York', datetime.datetime(2026, 6, 15, 12), 30,
795+
datetime.datetime(2026, 6, 15, 15, 59, 30),
796+
id='away-from-transitions',
797+
),
798+
])
799+
def test_cleanup_expiration_cutoff_across_timezones(self, tz_factory, zone, now, expires, expected_cutoff):
800+
self.app.conf.timezone = tz_factory(zone)
801+
self.app.conf.result_expires = expires
802+
self.backend = MongoBackend(app=self.app, url=self.default_url)
803+
self.backend.app.now = Mock(return_value=now.replace(tzinfo=self.app.timezone))
804+
self.backend.collection = Mock()
805+
self.backend.group_collection = Mock()
806+
807+
self.backend.cleanup()
808+
809+
# BSON converts aware datetimes to UTC, matching MongoDB comparisons.
810+
for collection in (self.backend.collection, self.backend.group_collection):
811+
collection.delete_many.assert_called_once()
812+
query = collection.delete_many.call_args[0][0]
813+
assert BSON.encode(query).decode() == {'date_done': {'$lt': expected_cutoff}}
814+
815+
@pytest.mark.parametrize('now,expires,expected_cutoff', [
816+
pytest.param(
817+
datetime.datetime(2026, 11, 1, 6, 30, tzinfo=datetime.timezone.utc),
818+
30,
819+
datetime.datetime(2026, 11, 1, 6, 29, 30),
820+
id='utc',
821+
),
822+
pytest.param(
823+
datetime.datetime(2026, 11, 1, 6, 30),
824+
30,
825+
datetime.datetime(2026, 11, 1, 6, 29, 30),
826+
id='naive-utc',
827+
),
828+
pytest.param(
829+
datetime.datetime(2026, 11, 1, 12, tzinfo=datetime.timezone(datetime.timedelta(hours=5, minutes=30))),
830+
30,
831+
datetime.datetime(2026, 11, 1, 6, 29, 30),
832+
id='fixed-offset',
833+
),
834+
pytest.param(
835+
datetime.datetime(2026, 11, 1, 6, 30, microsecond=125000, tzinfo=datetime.timezone.utc),
836+
0.125,
837+
datetime.datetime(2026, 11, 1, 6, 30),
838+
id='fractional-seconds',
839+
),
840+
pytest.param(
841+
datetime.datetime(2026, 11, 1, 6, 30, microsecond=1001, tzinfo=datetime.timezone.utc),
842+
0.0005,
843+
datetime.datetime(2026, 11, 1, 6, 30),
844+
id='bson-millisecond-precision',
845+
),
846+
pytest.param(
847+
datetime.datetime(2026, 11, 1, 6, 30, tzinfo=datetime.timezone.utc),
848+
-30,
849+
datetime.datetime(2026, 11, 1, 6, 30, 30),
850+
id='negative-expiration',
851+
),
852+
])
853+
def test_cleanup_expiration_cutoff_without_dst(self, now, expires, expected_cutoff):
854+
self.app.conf.result_expires = expires
855+
self.backend = MongoBackend(app=self.app, url=self.default_url)
856+
self.backend.app.now = Mock(return_value=now)
857+
self.backend.collection = Mock()
858+
self.backend.group_collection = Mock()
859+
860+
self.backend.cleanup()
861+
862+
# BSON converts aware datetimes to UTC, matching MongoDB comparisons.
863+
for collection in (self.backend.collection, self.backend.group_collection):
864+
collection.delete_many.assert_called_once()
865+
query = collection.delete_many.call_args[0][0]
866+
assert BSON.encode(query).decode() == {'date_done': {'$lt': expected_cutoff}}
867+
868+
@pytest.mark.parametrize('expires', [None, 0])
869+
def test_cleanup_without_expiration(self, expires):
870+
self.app.conf.result_expires = expires
871+
self.backend = MongoBackend(app=self.app, url=self.default_url)
872+
self.backend.app.now = Mock()
873+
self.backend.collection = Mock()
874+
self.backend.group_collection = Mock()
875+
876+
self.backend.cleanup()
877+
878+
self.backend.app.now.assert_not_called()
879+
self.backend.collection.delete_many.assert_not_called()
880+
self.backend.group_collection.delete_many.assert_not_called()
881+
715882
def test_prepare_client_options(self):
716883
with patch('pymongo.version_tuple', new=(3, 0, 3)):
717884
options = self.backend._prepare_client_options()

0 commit comments

Comments
 (0)