Skip to content

Commit 45a2f34

Browse files
committed
Address review: keep eager errors, let Reject through, cover after_return
- Re-raise from on_success in eager mode so apply() and task_always_eager still surface a broken hook. - Let Reject from the hook reach the outer handler as before. - Handle after_return the same way: it also runs after the outcome is recorded, and a raise there recorded a failure for a successful task, or a second failure for a failed one. - Assert the log call in the test and note which assertions pin the fix. - Document the behaviour in the task handler docs and the changelog.
1 parent fd71acf commit 45a2f34

4 files changed

Lines changed: 96 additions & 7 deletions

File tree

‎Changelog.rst‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,13 @@ an overview of what's new in Celery 5.6.
1212
Unreleased
1313
==========
1414

15+
- An exception raised by ``Task.on_success`` or ``Task.after_return`` in a
16+
worker is now logged at ``ERROR`` level instead of being recorded as a task
17+
failure. The task's errbacks no longer run for a task that succeeded, and a
18+
failed task is no longer recorded a second time. Broken hooks therefore no
19+
longer send the ``task_failure`` or ``task_internal_error`` signals. Eager
20+
execution still propagates the exception (#10724).
21+
1522
- Fix ``%%`` in node and host format strings to produce a literal percent sign.
1623
Previously ``%%h``, ``%%n`` and ``%%d`` expanded the variable after an extra
1724
percent sign. Log and pid file paths passed through ``celery multi`` are formatted

‎celery/app/trace.py‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -635,10 +635,15 @@ def trace_task(uuid, args, kwargs, request=None):
635635
if task_on_success:
636636
try:
637637
task_on_success(retval, uuid, args, kwargs)
638+
except Reject:
639+
raise
638640
except Exception as exc:
639-
# SUCCESS is already stored: recording a
641+
# The outcome is already stored: recording a
640642
# failure now would fire the errbacks for a
641-
# task that succeeded.
643+
# task that succeeded. Eager callers (tests,
644+
# task_always_eager) still see the error.
645+
if eager:
646+
raise
642647
logger.exception(
643648
'Task %s[%s] on_success handler raised: %r',
644649
get_task_name(task_request, name),
@@ -660,9 +665,23 @@ def trace_task(uuid, args, kwargs, request=None):
660665
# -* POST *-
661666
if state not in IGNORE_STATES:
662667
if task_after_return:
663-
task_after_return(
664-
state, retval, uuid, args, kwargs, None,
665-
)
668+
try:
669+
task_after_return(
670+
state, retval, uuid, args, kwargs, None,
671+
)
672+
except Reject:
673+
raise
674+
except Exception as exc:
675+
# Same as on_success: the outcome is already
676+
# recorded, so don't record a second one.
677+
if eager:
678+
raise
679+
logger.exception(
680+
'Task %s[%s] after_return handler raised: %r',
681+
get_task_name(task_request, name),
682+
uuid, exc,
683+
)
684+
traceback_clear(exc)
666685
finally:
667686
try:
668687
if postrun_receivers:

‎docs/userguide/tasks.rst‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1731,6 +1731,12 @@ Available handlers
17311731

17321732
The return value of this handler is ignored.
17331733

1734+
If this handler raises, the worker logs the exception at ``ERROR`` level
1735+
and the task stays ``SUCCESS``: errbacks don't run and the
1736+
:signal:`task_failure` and :signal:`task_internal_error` signals aren't
1737+
sent. When the task runs eagerly (``apply()``, :setting:`task_always_eager`)
1738+
the exception propagates to the caller.
1739+
17341740
.. method:: on_retry(self, exc, task_id, args, kwargs, einfo)
17351741

17361742
Retry handler.
@@ -1790,6 +1796,10 @@ Available handlers
17901796

17911797
The return value of this handler is ignored.
17921798

1799+
If this handler raises, the worker logs the exception at ``ERROR`` level
1800+
and the outcome that was already recorded stands. When the task runs
1801+
eagerly the exception propagates to the caller.
1802+
17931803
Example usage
17941804
~~~~~~~~~~~~~
17951805

‎t/unit/tasks/test_trace.py‎

Lines changed: 55 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,8 @@ def add_with_failing_success(x, y):
8484
on_task_success = Mock()
8585
signals.task_success.connect(on_task_success)
8686
try:
87-
with patch('celery.app.trace.TraceInfo.handle_failure') as handle_failure:
87+
with patch('celery.app.trace.TraceInfo.handle_failure') as handle_failure, \
88+
patch('celery.app.trace.logger') as mock_logger:
8889
retval, info, _ = self.trace(
8990
add_with_failing_success, (2, 2), {}, eager=False,
9091
)
@@ -93,12 +94,64 @@ def add_with_failing_success(x, y):
9394

9495
assert info is None
9596
assert retval == 4
96-
handle_failure.assert_not_called()
97+
# The cache backend keeps a stored SUCCESS even on main, so the state
98+
# check alone doesn't pin the fix: handle_failure, the task_success
99+
# receiver and after_return do.
97100
assert add_with_failing_success.backend.get_state('id-1') == states.SUCCESS
101+
handle_failure.assert_not_called()
98102
on_task_success.assert_called()
99103
add_with_failing_success.after_return.assert_called_with(
100104
states.SUCCESS, 4, 'id-1', (2, 2), {}, None,
101105
)
106+
mock_logger.exception.assert_called_once()
107+
assert 'on_success handler raised' in mock_logger.exception.call_args[0][0]
108+
109+
def test_trace_on_success_raises_eager_propagates(self):
110+
@self.app.task(shared=False,
111+
on_success=Mock(side_effect=RuntimeError('hook failed')))
112+
def add_with_failing_success(x, y):
113+
return x + y
114+
115+
with pytest.raises(RuntimeError, match='hook failed'):
116+
self.trace(add_with_failing_success, (2, 2), {}, eager=True)
117+
118+
def test_trace_on_success_raises_reject_is_not_swallowed(self):
119+
@self.app.task(shared=False,
120+
on_success=Mock(side_effect=Reject('no', requeue=False)))
121+
def add_with_rejecting_success(x, y):
122+
return x + y
123+
124+
with pytest.raises(Reject):
125+
self.trace(add_with_rejecting_success, (2, 2), {}, eager=False)
126+
127+
@pytest.mark.parametrize('fails', [False, True])
128+
def test_trace_after_return_raises(self, fails):
129+
@self.app.task(shared=False,
130+
after_return=Mock(side_effect=RuntimeError('hook failed')))
131+
def task_with_failing_after_return(x, y):
132+
if fails:
133+
raise KeyError('task failed')
134+
return x + y
135+
136+
with patch('celery.app.trace.TraceInfo.handle_failure',
137+
return_value=ExceptionInfo) as handle_failure, \
138+
patch('celery.app.trace.logger') as mock_logger:
139+
self.trace(task_with_failing_after_return, (2, 2), {}, eager=False)
140+
141+
# A failed task is recorded once, by its own exception; a successful
142+
# one is not recorded as failed at all.
143+
assert handle_failure.call_count == (1 if fails else 0)
144+
mock_logger.exception.assert_called_once()
145+
assert 'after_return handler raised' in mock_logger.exception.call_args[0][0]
146+
147+
def test_trace_after_return_raises_eager_propagates(self):
148+
@self.app.task(shared=False,
149+
after_return=Mock(side_effect=RuntimeError('hook failed')))
150+
def add_with_failing_after_return(x, y):
151+
return x + y
152+
153+
with pytest.raises(RuntimeError, match='hook failed'):
154+
self.trace(add_with_failing_after_return, (2, 2), {}, eager=True)
102155

103156
def test_get_log_policy(self):
104157
einfo = Mock(name='einfo')

0 commit comments

Comments
 (0)