|
22 | 22 | import functools |
23 | 23 | import json |
24 | 24 | import os |
| 25 | +import signal |
25 | 26 | import textwrap |
26 | 27 | import time |
27 | 28 | from collections.abc import Iterable |
@@ -6982,3 +6983,200 @@ def test_stats_tags_with_standalone_and_key_value_tags(create_runtime_ti): |
6982 | 6983 | "task_id": "t", |
6983 | 6984 | "run_type": "manual", |
6984 | 6985 | } |
| 6986 | + |
| 6987 | + |
| 6988 | +class TestSigtermTerminatesTask: |
| 6989 | + """ |
| 6990 | + A SIGTERM must not leave a task that was killed mid-execute committed as successful. |
| 6991 | +
|
| 6992 | + Regression tests for #73006. `_on_term` calls `on_kill()` and, before this was |
| 6993 | + fixed, returned, so `execute()` resumed from where the signal interrupted it and ran |
| 6994 | + to completion. Any operator whose `on_kill` destroys the work it is waiting on -- |
| 6995 | + `KubernetesPodOperator` deletes its pod -- then returned normally and the task |
| 6996 | + instance was committed `success`. |
| 6997 | +
|
| 6998 | + `run()` works on its own copy of the task, so the assertions read `ti.task` rather |
| 6999 | + than the object each test builds. |
| 7000 | + """ |
| 7001 | + |
| 7002 | + @pytest.fixture(autouse=True) |
| 7003 | + def _restore_sigterm_handler(self): |
| 7004 | + """`run()` installs its own handler in this process, so put the old one back.""" |
| 7005 | + previous = signal.getsignal(signal.SIGTERM) |
| 7006 | + yield |
| 7007 | + signal.signal(signal.SIGTERM, previous) |
| 7008 | + |
| 7009 | + def test_sigterm_during_execute_fails_the_task(self, create_runtime_ti, mock_supervisor_comms): |
| 7010 | + """The task fails with `AirflowTaskTerminated` and `execute` does not reach its end.""" |
| 7011 | + |
| 7012 | + class KilledOperator(BaseOperator): |
| 7013 | + def __init__(self, **kwargs): |
| 7014 | + super().__init__(**kwargs) |
| 7015 | + self.killed = False |
| 7016 | + self.reached_end_of_execute = False |
| 7017 | + |
| 7018 | + def execute(self, context): |
| 7019 | + os.kill(os.getpid(), signal.SIGTERM) |
| 7020 | + # Python runs the handler at the next bytecode boundary. Before the fix |
| 7021 | + # the handler returned and this line ran, which is what committed the |
| 7022 | + # task successful after its work had already been destroyed. |
| 7023 | + self.reached_end_of_execute = True |
| 7024 | + |
| 7025 | + def on_kill(self): |
| 7026 | + self.killed = True |
| 7027 | + |
| 7028 | + ti = create_runtime_ti(task=KilledOperator(task_id="killed")) |
| 7029 | + |
| 7030 | + state, _, error = run(ti, ti.get_template_context(), log=mock.MagicMock()) |
| 7031 | + |
| 7032 | + assert ti.task.killed is True |
| 7033 | + assert ti.task.reached_end_of_execute is False |
| 7034 | + assert state == TaskInstanceState.FAILED |
| 7035 | + assert isinstance(error, AirflowTaskTerminated) |
| 7036 | + |
| 7037 | + def test_pod_operator_shape_is_not_committed_successful(self, create_runtime_ti, mock_supervisor_comms): |
| 7038 | + """ |
| 7039 | + The `KubernetesPodOperator` shape from the issue. |
| 7040 | +
|
| 7041 | + `on_kill` tears the pod down and sets a flag, and the operator's own cleanup then |
| 7042 | + declines to raise because that flag is set (the `_killed` guard from #36749). |
| 7043 | + Nothing in the operator fails the task, so the interrupt has to come from the |
| 7044 | + runner. |
| 7045 | + """ |
| 7046 | + |
| 7047 | + class PodOperatorShape(BaseOperator): |
| 7048 | + def __init__(self, **kwargs): |
| 7049 | + super().__init__(**kwargs) |
| 7050 | + self._killed = False |
| 7051 | + self.pod_deleted = False |
| 7052 | + |
| 7053 | + def execute(self, context): |
| 7054 | + os.kill(os.getpid(), signal.SIGTERM) |
| 7055 | + self.cleanup() |
| 7056 | + return "xcom-value" |
| 7057 | + |
| 7058 | + def cleanup(self): |
| 7059 | + if self._killed: |
| 7060 | + return |
| 7061 | + # The operator's own failure raise, which the guard above skips. |
| 7062 | + raise RuntimeError("pod did not reach Succeeded") |
| 7063 | + |
| 7064 | + def on_kill(self): |
| 7065 | + self._killed = True |
| 7066 | + self.pod_deleted = True |
| 7067 | + |
| 7068 | + ti = create_runtime_ti(task=PodOperatorShape(task_id="pod")) |
| 7069 | + |
| 7070 | + state, _, error = run(ti, ti.get_template_context(), log=mock.MagicMock()) |
| 7071 | + |
| 7072 | + assert ti.task.pod_deleted is True |
| 7073 | + assert state == TaskInstanceState.FAILED |
| 7074 | + assert isinstance(error, AirflowTaskTerminated) |
| 7075 | + |
| 7076 | + def test_on_kill_runs_once_when_a_second_sigterm_arrives(self, create_runtime_ti, mock_supervisor_comms): |
| 7077 | + """ |
| 7078 | + A SIGTERM that lands while the first one is still unwinding is ignored. |
| 7079 | +
|
| 7080 | + Without the guard an `on_kill` that is itself interrupted re-enters the handler, |
| 7081 | + so `on_kill` runs hundreds of times before the stack unwinds. |
| 7082 | + """ |
| 7083 | + |
| 7084 | + class ResignallingOperator(BaseOperator): |
| 7085 | + def __init__(self, **kwargs): |
| 7086 | + super().__init__(**kwargs) |
| 7087 | + self.on_kill_calls = 0 |
| 7088 | + |
| 7089 | + def execute(self, context): |
| 7090 | + os.kill(os.getpid(), signal.SIGTERM) |
| 7091 | + |
| 7092 | + def on_kill(self): |
| 7093 | + self.on_kill_calls += 1 |
| 7094 | + # A supervisor escalating its kill sends another SIGTERM while `on_kill` |
| 7095 | + # is still running. |
| 7096 | + os.kill(os.getpid(), signal.SIGTERM) |
| 7097 | + |
| 7098 | + ti = create_runtime_ti(task=ResignallingOperator(task_id="resignalled")) |
| 7099 | + |
| 7100 | + state, _, error = run(ti, ti.get_template_context(), log=mock.MagicMock()) |
| 7101 | + |
| 7102 | + assert ti.task.on_kill_calls == 1 |
| 7103 | + assert state == TaskInstanceState.FAILED |
| 7104 | + assert isinstance(error, AirflowTaskTerminated) |
| 7105 | + |
| 7106 | + def test_failing_on_kill_still_terminates_the_task(self, create_runtime_ti, mock_supervisor_comms): |
| 7107 | + """An `on_kill` that raises must not mask the termination.""" |
| 7108 | + |
| 7109 | + class BrokenOnKillOperator(BaseOperator): |
| 7110 | + def __init__(self, **kwargs): |
| 7111 | + super().__init__(**kwargs) |
| 7112 | + self.reached_end_of_execute = False |
| 7113 | + |
| 7114 | + def execute(self, context): |
| 7115 | + os.kill(os.getpid(), signal.SIGTERM) |
| 7116 | + self.reached_end_of_execute = True |
| 7117 | + |
| 7118 | + def on_kill(self): |
| 7119 | + raise RuntimeError("could not reach the cluster to delete the pod") |
| 7120 | + |
| 7121 | + ti = create_runtime_ti(task=BrokenOnKillOperator(task_id="broken_on_kill")) |
| 7122 | + |
| 7123 | + state, _, error = run(ti, ti.get_template_context(), log=mock.MagicMock()) |
| 7124 | + |
| 7125 | + assert ti.task.reached_end_of_execute is False |
| 7126 | + assert state == TaskInstanceState.FAILED |
| 7127 | + assert isinstance(error, AirflowTaskTerminated) |
| 7128 | + |
| 7129 | + def test_sigterm_after_execute_returns_keeps_the_task_successful( |
| 7130 | + self, create_runtime_ti, mock_supervisor_comms, monkeypatch |
| 7131 | + ): |
| 7132 | + """ |
| 7133 | + The supervisor's `task_success_overtime` kill arrives after the task has finished. |
| 7134 | +
|
| 7135 | + The result is already there, so `on_kill` still runs but the task stays |
| 7136 | + successful rather than being turned into a failure. |
| 7137 | + """ |
| 7138 | + real_push = task_runner._push_xcom_if_needed |
| 7139 | + killed = {} |
| 7140 | + |
| 7141 | + def push_then_signal(result, ti, log): |
| 7142 | + os.kill(os.getpid(), signal.SIGTERM) |
| 7143 | + return real_push(result, ti, log) |
| 7144 | + |
| 7145 | + monkeypatch.setattr(task_runner, "_push_xcom_if_needed", push_then_signal) |
| 7146 | + |
| 7147 | + class FinishedOperator(BaseOperator): |
| 7148 | + def execute(self, context): |
| 7149 | + return "done" |
| 7150 | + |
| 7151 | + def on_kill(self): |
| 7152 | + killed["called"] = True |
| 7153 | + |
| 7154 | + ti = create_runtime_ti(task=FinishedOperator(task_id="finished")) |
| 7155 | + |
| 7156 | + state, _, error = run(ti, ti.get_template_context(), log=mock.MagicMock()) |
| 7157 | + |
| 7158 | + assert killed.get("called") is True |
| 7159 | + assert error is None |
| 7160 | + assert state == TaskInstanceState.SUCCESS |
| 7161 | + |
| 7162 | + def test_task_without_sigterm_is_unaffected(self, create_runtime_ti, mock_supervisor_comms): |
| 7163 | + """The control: no signal, so nothing changes.""" |
| 7164 | + |
| 7165 | + class QuietOperator(BaseOperator): |
| 7166 | + def __init__(self, **kwargs): |
| 7167 | + super().__init__(**kwargs) |
| 7168 | + self.killed = False |
| 7169 | + |
| 7170 | + def execute(self, context): |
| 7171 | + return "done" |
| 7172 | + |
| 7173 | + def on_kill(self): |
| 7174 | + self.killed = True |
| 7175 | + |
| 7176 | + ti = create_runtime_ti(task=QuietOperator(task_id="quiet")) |
| 7177 | + |
| 7178 | + state, _, error = run(ti, ti.get_template_context(), log=mock.MagicMock()) |
| 7179 | + |
| 7180 | + assert ti.task.killed is False |
| 7181 | + assert error is None |
| 7182 | + assert state == TaskInstanceState.SUCCESS |
0 commit comments