Skip to content

Commit 3c12f12

Browse files
Fail a task whose execute is interrupted by SIGTERM
The task runner's SIGTERM handler called on_kill() and returned, so execute() resumed from where the signal interrupted it and ran to completion. An operator whose on_kill destroys the work it is waiting on then returned normally and the task instance was committed success. KubernetesPodOperator deletes its pod in on_kill and its cleanup() declines to raise because _killed is set, so a pod evicted mid-run, or a worker that is drained, produced a green task whose work never happened. AirflowTaskTerminated was already caught in the runner but nothing raised it anywhere; the raise was dropped when the handler was ported. It is raised again, after on_kill, but only while the task's own body is on the stack: the supervisor also sends SIGTERM once a task has reported success and is in task_success_overtime, and that must not turn a finished task into a failed one. A second SIGTERM arriving while the first is unwinding is ignored, so on_kill runs once rather than re-entering the handler, and an on_kill that raises no longer hides the termination. Closes apache#73006
1 parent 5428b3b commit 3c12f12

2 files changed

Lines changed: 232 additions & 2 deletions

File tree

‎task-sdk/src/airflow/sdk/execution_time/task_runner.py‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1565,13 +1565,36 @@ def _run_task_and_map_outcome(
15651565
assert isinstance(ti.task, BaseOperator)
15661566

15671567
parent_pid = os.getpid()
1568+
# Whether the task's own body is on the stack. A SIGTERM is only allowed to
1569+
# interrupt the task while this is true: the supervisor also sends SIGTERM once a
1570+
# task has already reported success and is in `task_success_overtime`, and turning
1571+
# that into a failure would lose a result the server has already accepted.
1572+
in_task_body = False
1573+
# Set once the interrupt is in flight, so a second SIGTERM arriving while the first
1574+
# is still unwinding does not run `on_kill` twice or replace the exception that is
1575+
# already propagating with an identical one.
1576+
terminating = False
15681577

15691578
def _on_term(signum, frame):
1579+
nonlocal terminating
15701580
pid = os.getpid()
15711581
if pid != parent_pid:
15721582
return
1583+
if terminating:
1584+
return
15731585

1574-
ti.task.on_kill()
1586+
if in_task_body:
1587+
terminating = True
1588+
1589+
try:
1590+
ti.task.on_kill()
1591+
finally:
1592+
if terminating:
1593+
# Airflow 2 raised this right after `on_kill` so that a terminated task
1594+
# could not be committed as successful. The port in #50141 dropped the
1595+
# raise and kept the handler, so `execute()` simply resumed and the task
1596+
# finished normally even though `on_kill` had already destroyed its work.
1597+
raise AirflowTaskTerminated(f"Task received SIGTERM signal {signum}")
15751598

15761599
signal.signal(signal.SIGTERM, _on_term)
15771600

@@ -1601,7 +1624,16 @@ def _on_term(signum, frame):
16011624
return state, msg, error
16021625

16031626
try:
1604-
result = _execute_task(context=context, ti=ti, log=log)
1627+
in_task_body = True
1628+
try:
1629+
result = _execute_task(context=context, ti=ti, log=log)
1630+
finally:
1631+
# `finally` rather than a reset on each branch below, so that the
1632+
# ones that raise a `BaseException` -- `AirflowTaskTimeout`,
1633+
# `SystemExit` -- also leave the window. A SIGTERM that arrives
1634+
# while one of those is being handled belongs to the outcome that
1635+
# is already in flight.
1636+
in_task_body = False
16051637
log.info("::group::Post Execute")
16061638
except Exception:
16071639
import jinja2

‎task-sdk/tests/task_sdk/execution_time/test_task_runner.py‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import functools
2323
import json
2424
import os
25+
import signal
2526
import textwrap
2627
import time
2728
from collections.abc import Iterable
@@ -6982,3 +6983,200 @@ def test_stats_tags_with_standalone_and_key_value_tags(create_runtime_ti):
69826983
"task_id": "t",
69836984
"run_type": "manual",
69846985
}
6986+
6987+
6988+
class TestSigtermTerminatesTask:
6989+
"""
6990+
A SIGTERM must not leave a task that was killed mid-execute committed as successful.
6991+
6992+
Regression tests for #73006. `_on_term` calls `on_kill()` and, before this was
6993+
fixed, returned, so `execute()` resumed from where the signal interrupted it and ran
6994+
to completion. Any operator whose `on_kill` destroys the work it is waiting on --
6995+
`KubernetesPodOperator` deletes its pod -- then returned normally and the task
6996+
instance was committed `success`.
6997+
6998+
`run()` works on its own copy of the task, so the assertions read `ti.task` rather
6999+
than the object each test builds.
7000+
"""
7001+
7002+
@pytest.fixture(autouse=True)
7003+
def _restore_sigterm_handler(self):
7004+
"""`run()` installs its own handler in this process, so put the old one back."""
7005+
previous = signal.getsignal(signal.SIGTERM)
7006+
yield
7007+
signal.signal(signal.SIGTERM, previous)
7008+
7009+
def test_sigterm_during_execute_fails_the_task(self, create_runtime_ti, mock_supervisor_comms):
7010+
"""The task fails with `AirflowTaskTerminated` and `execute` does not reach its end."""
7011+
7012+
class KilledOperator(BaseOperator):
7013+
def __init__(self, **kwargs):
7014+
super().__init__(**kwargs)
7015+
self.killed = False
7016+
self.reached_end_of_execute = False
7017+
7018+
def execute(self, context):
7019+
os.kill(os.getpid(), signal.SIGTERM)
7020+
# Python runs the handler at the next bytecode boundary. Before the fix
7021+
# the handler returned and this line ran, which is what committed the
7022+
# task successful after its work had already been destroyed.
7023+
self.reached_end_of_execute = True
7024+
7025+
def on_kill(self):
7026+
self.killed = True
7027+
7028+
ti = create_runtime_ti(task=KilledOperator(task_id="killed"))
7029+
7030+
state, _, error = run(ti, ti.get_template_context(), log=mock.MagicMock())
7031+
7032+
assert ti.task.killed is True
7033+
assert ti.task.reached_end_of_execute is False
7034+
assert state == TaskInstanceState.FAILED
7035+
assert isinstance(error, AirflowTaskTerminated)
7036+
7037+
def test_pod_operator_shape_is_not_committed_successful(self, create_runtime_ti, mock_supervisor_comms):
7038+
"""
7039+
The `KubernetesPodOperator` shape from the issue.
7040+
7041+
`on_kill` tears the pod down and sets a flag, and the operator's own cleanup then
7042+
declines to raise because that flag is set (the `_killed` guard from #36749).
7043+
Nothing in the operator fails the task, so the interrupt has to come from the
7044+
runner.
7045+
"""
7046+
7047+
class PodOperatorShape(BaseOperator):
7048+
def __init__(self, **kwargs):
7049+
super().__init__(**kwargs)
7050+
self._killed = False
7051+
self.pod_deleted = False
7052+
7053+
def execute(self, context):
7054+
os.kill(os.getpid(), signal.SIGTERM)
7055+
self.cleanup()
7056+
return "xcom-value"
7057+
7058+
def cleanup(self):
7059+
if self._killed:
7060+
return
7061+
# The operator's own failure raise, which the guard above skips.
7062+
raise RuntimeError("pod did not reach Succeeded")
7063+
7064+
def on_kill(self):
7065+
self._killed = True
7066+
self.pod_deleted = True
7067+
7068+
ti = create_runtime_ti(task=PodOperatorShape(task_id="pod"))
7069+
7070+
state, _, error = run(ti, ti.get_template_context(), log=mock.MagicMock())
7071+
7072+
assert ti.task.pod_deleted is True
7073+
assert state == TaskInstanceState.FAILED
7074+
assert isinstance(error, AirflowTaskTerminated)
7075+
7076+
def test_on_kill_runs_once_when_a_second_sigterm_arrives(self, create_runtime_ti, mock_supervisor_comms):
7077+
"""
7078+
A SIGTERM that lands while the first one is still unwinding is ignored.
7079+
7080+
Without the guard an `on_kill` that is itself interrupted re-enters the handler,
7081+
so `on_kill` runs hundreds of times before the stack unwinds.
7082+
"""
7083+
7084+
class ResignallingOperator(BaseOperator):
7085+
def __init__(self, **kwargs):
7086+
super().__init__(**kwargs)
7087+
self.on_kill_calls = 0
7088+
7089+
def execute(self, context):
7090+
os.kill(os.getpid(), signal.SIGTERM)
7091+
7092+
def on_kill(self):
7093+
self.on_kill_calls += 1
7094+
# A supervisor escalating its kill sends another SIGTERM while `on_kill`
7095+
# is still running.
7096+
os.kill(os.getpid(), signal.SIGTERM)
7097+
7098+
ti = create_runtime_ti(task=ResignallingOperator(task_id="resignalled"))
7099+
7100+
state, _, error = run(ti, ti.get_template_context(), log=mock.MagicMock())
7101+
7102+
assert ti.task.on_kill_calls == 1
7103+
assert state == TaskInstanceState.FAILED
7104+
assert isinstance(error, AirflowTaskTerminated)
7105+
7106+
def test_failing_on_kill_still_terminates_the_task(self, create_runtime_ti, mock_supervisor_comms):
7107+
"""An `on_kill` that raises must not mask the termination."""
7108+
7109+
class BrokenOnKillOperator(BaseOperator):
7110+
def __init__(self, **kwargs):
7111+
super().__init__(**kwargs)
7112+
self.reached_end_of_execute = False
7113+
7114+
def execute(self, context):
7115+
os.kill(os.getpid(), signal.SIGTERM)
7116+
self.reached_end_of_execute = True
7117+
7118+
def on_kill(self):
7119+
raise RuntimeError("could not reach the cluster to delete the pod")
7120+
7121+
ti = create_runtime_ti(task=BrokenOnKillOperator(task_id="broken_on_kill"))
7122+
7123+
state, _, error = run(ti, ti.get_template_context(), log=mock.MagicMock())
7124+
7125+
assert ti.task.reached_end_of_execute is False
7126+
assert state == TaskInstanceState.FAILED
7127+
assert isinstance(error, AirflowTaskTerminated)
7128+
7129+
def test_sigterm_after_execute_returns_keeps_the_task_successful(
7130+
self, create_runtime_ti, mock_supervisor_comms, monkeypatch
7131+
):
7132+
"""
7133+
The supervisor's `task_success_overtime` kill arrives after the task has finished.
7134+
7135+
The result is already there, so `on_kill` still runs but the task stays
7136+
successful rather than being turned into a failure.
7137+
"""
7138+
real_push = task_runner._push_xcom_if_needed
7139+
killed = {}
7140+
7141+
def push_then_signal(result, ti, log):
7142+
os.kill(os.getpid(), signal.SIGTERM)
7143+
return real_push(result, ti, log)
7144+
7145+
monkeypatch.setattr(task_runner, "_push_xcom_if_needed", push_then_signal)
7146+
7147+
class FinishedOperator(BaseOperator):
7148+
def execute(self, context):
7149+
return "done"
7150+
7151+
def on_kill(self):
7152+
killed["called"] = True
7153+
7154+
ti = create_runtime_ti(task=FinishedOperator(task_id="finished"))
7155+
7156+
state, _, error = run(ti, ti.get_template_context(), log=mock.MagicMock())
7157+
7158+
assert killed.get("called") is True
7159+
assert error is None
7160+
assert state == TaskInstanceState.SUCCESS
7161+
7162+
def test_task_without_sigterm_is_unaffected(self, create_runtime_ti, mock_supervisor_comms):
7163+
"""The control: no signal, so nothing changes."""
7164+
7165+
class QuietOperator(BaseOperator):
7166+
def __init__(self, **kwargs):
7167+
super().__init__(**kwargs)
7168+
self.killed = False
7169+
7170+
def execute(self, context):
7171+
return "done"
7172+
7173+
def on_kill(self):
7174+
self.killed = True
7175+
7176+
ti = create_runtime_ti(task=QuietOperator(task_id="quiet"))
7177+
7178+
state, _, error = run(ti, ti.get_template_context(), log=mock.MagicMock())
7179+
7180+
assert ti.task.killed is False
7181+
assert error is None
7182+
assert state == TaskInstanceState.SUCCESS

0 commit comments

Comments
 (0)