Skip to content

Commit 78a66bd

Browse files
committed
Don't fail a task whose SUCCESS was stored before the worker was lost
When the pool child stores SUCCESS and then dies (OOM kill, SIGKILL, a crash in on_success, a task_postrun receiver or process_cleanup), the parent gets WorkerLostError and Request.on_failure calls mark_as_failure unconditionally. That fires the errbacks for a task that succeeded, counts a chord header twice, and with reject_on_worker_lost requeues a task that already ran. Check the stored state first. If it is SUCCESS, acknowledge (acks_late) and return without recording a failure. If the backend can't be read, keep the existing failure handling. Refs #10724
1 parent 6bc4220 commit 78a66bd

2 files changed

Lines changed: 69 additions & 0 deletions

File tree

‎celery/worker/request.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -705,6 +705,17 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False):
705705
# (acks_late) acknowledge after result stored.
706706
requeue = False
707707
is_worker_lost = isinstance(exc, WorkerLostError)
708+
if is_worker_lost and self._success_was_stored():
709+
# The pool child stored SUCCESS and was lost afterwards, e.g. in
710+
# on_success, a task_postrun receiver or process_cleanup.
711+
# Recording a failure here would fire the errbacks and count a
712+
# chord header twice, and requeueing would run the task again.
713+
warn('Task %s[%s] succeeded before the worker was lost: %r',
714+
self.name, self.id, exc)
715+
if self.task.acks_late:
716+
self.acknowledge()
717+
return
718+
708719
if self.task.acks_late:
709720
is_timeout = isinstance(exc, TimeLimitExceeded)
710721
ack_flag = self.task.acks_on_timeout if is_timeout else self.task.acks_on_failure
@@ -761,6 +772,13 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False):
761772
error('Task handler raised error: %r', exc,
762773
exc_info=exc_info.exc_info)
763774

775+
def _success_was_stored(self):
776+
try:
777+
return self.task.backend.get_state(self.id) == states.SUCCESS
778+
except Exception: # pylint: disable=broad-except
779+
# No backend, or it can't be reached: keep the failure handling.
780+
return False
781+
764782
def _worker_lost_info(self, exc):
765783
if isinstance(exc, SystemExit):
766784
code = exc.code

‎t/unit/worker/test_request.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -962,6 +962,57 @@ def get_ei():
962962
job.on_failure(exc_info)
963963
assert self.mytask.backend.get_status(job.id) == states.PENDING
964964

965+
def test_on_failure_WorkerLostError_after_success_was_stored(self):
966+
errbacks = []
967+
968+
@self.app.task(shared=False)
969+
def report_failure(request, exc, traceback):
970+
errbacks.append(type(exc).__name__)
971+
972+
job = self.xRequest(errbacks=[report_failure.s()])
973+
# The child stored the result, then died before handing it to the parent.
974+
self.mytask.backend.mark_as_done(job.id, 1)
975+
try:
976+
raise WorkerLostError('Worker exited prematurely: exitcode 1.')
977+
except WorkerLostError:
978+
exc_info = ExceptionInfo()
979+
job.on_failure(exc_info)
980+
981+
assert self.mytask.backend.get_status(job.id) == states.SUCCESS
982+
assert errbacks == []
983+
984+
def test_on_failure_WorkerLostError_after_success_was_stored_acks_late(self):
985+
try:
986+
raise WorkerLostError()
987+
except WorkerLostError:
988+
einfo = ExceptionInfo(internal=True)
989+
990+
req = self.get_request(self.add.s(2, 2))
991+
req.task.acks_late = True
992+
req.task.reject_on_worker_lost = True
993+
req.task.backend = Mock()
994+
req.task.backend.get_state.return_value = states.SUCCESS
995+
996+
req.on_failure(einfo)
997+
998+
req.on_ack.assert_called_with(req_logger, req.connection_errors)
999+
req.on_reject.assert_not_called()
1000+
req.task.backend.mark_as_failure.assert_not_called()
1001+
1002+
def test_on_failure_WorkerLostError_backend_unavailable(self):
1003+
try:
1004+
raise WorkerLostError()
1005+
except WorkerLostError:
1006+
einfo = ExceptionInfo(internal=True)
1007+
1008+
req = self.get_request(self.add.s(2, 2))
1009+
req.task.backend = Mock()
1010+
req.task.backend.get_state.side_effect = ConnectionError()
1011+
1012+
req.on_failure(einfo)
1013+
1014+
req.task.backend.mark_as_failure.assert_called_once()
1015+
9651016
def test_on_failure_acks_late_reject_on_worker_lost_enabled(self):
9661017
try:
9671018
raise WorkerLostError()

0 commit comments

Comments
 (0)