Skip to content

Commit 4f76227

Browse files
authored
Forward termination signals from supervisor to task subprocess (#61627)
When a Kubernetes worker pod receives SIGTERM (spot interruption, scaling down, rolling update), the signal is delivered to the supervisor process (PID 1 in the container). The supervisor has no signal handler installed and exits with default behavior, leaving the task subprocess orphaned without ever calling the operator's `on_kill()` hook. Spawned resources (pods, subprocesses, etc.) are never cleaned up. The task subprocess already has a SIGTERM handler registered in `task_runner.py` that calls `on_kill()`, but the signal never reaches it because the supervisor process terminates first. Install SIGTERM/SIGINT signal handlers in `ActivitySubprocess.wait()` that forward the received signal to the task subprocess via `os.kill()`. The child's existing handler then calls `on_kill()` as expected, restoring the Airflow 2 behavior. Handlers are saved before `_monitor_subprocess()` and restored in a `finally` block. Placing the handler in `ActivitySubprocess.wait()` (rather than `supervise_task`) makes the forwarding work for any coordinator that uses `ActivitySubprocess` and keeps the handler colocated with the process it forwards to. Fixes: #58936 Signed-off-by: André Ahlert <[email protected]>
1 parent 120b49a commit 4f76227

3 files changed

Lines changed: 115 additions & 1 deletion

File tree

‎task-sdk/src/airflow/sdk/execution_time/supervisor.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1391,10 +1391,31 @@ def wait(self) -> int:
13911391
if self._exit_code is not None:
13921392
return self._exit_code
13931393

1394+
# Forward termination signals to the task subprocess so the operator's
1395+
# on_kill() hook runs on graceful shutdown (e.g. K8s pod SIGTERM).
1396+
# Without this the supervisor exits on SIGTERM without notifying the
1397+
# child, leaving spawned resources (pods, subprocesses, etc.) running.
1398+
prev_sigterm = signal.getsignal(signal.SIGTERM)
1399+
prev_sigint = signal.getsignal(signal.SIGINT)
1400+
1401+
def _forward_signal(signum, frame):
1402+
log.info(
1403+
"Received signal, forwarding to task subprocess",
1404+
signal=signal.Signals(signum).name,
1405+
pid=self.pid,
1406+
)
1407+
with suppress(ProcessLookupError):
1408+
os.kill(self.pid, signum)
1409+
1410+
signal.signal(signal.SIGTERM, _forward_signal)
1411+
signal.signal(signal.SIGINT, _forward_signal)
1412+
13941413
try:
13951414
self._monitor_subprocess()
13961415
finally:
13971416
self.selector.close()
1417+
signal.signal(signal.SIGTERM, prev_sigterm)
1418+
signal.signal(signal.SIGINT, prev_sigint)
13981419

13991420
# self._monitor_subprocess() will set the exit code when the process has finished
14001421
# If it hasn't, assume it's failed
Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
from __future__ import annotations
18+
19+
import os
20+
import signal
21+
import time
22+
23+
from airflow.sdk.bases.operator import BaseOperator
24+
from airflow.sdk.definitions.dag import dag
25+
26+
27+
class SignalForwardOperator(BaseOperator):
28+
"""Send SIGTERM to the supervisor parent process to exercise signal forwarding."""
29+
30+
def execute(self, context):
31+
print("EXECUTE_STARTED", flush=True)
32+
os.kill(os.getppid(), signal.SIGTERM)
33+
time.sleep(2)
34+
35+
def on_kill(self) -> None:
36+
print("ON_KILL_CALLED_VIA_SIGNAL_FORWARDING", flush=True)
37+
38+
39+
@dag()
40+
def signal_forward_test():
41+
SignalForwardOperator(task_id="signal_task")
42+
43+
44+
signal_forward_test()

‎task-sdk/tests/task_sdk/execution_time/test_supervisor.py‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,9 @@
2727
import socket
2828
import subprocess
2929
import sys
30+
import threading
3031
import time
31-
from contextlib import nullcontext
32+
from contextlib import nullcontext, suppress
3233
from dataclasses import dataclass, field
3334
from datetime import datetime, timezone as dt_timezone
3435
from operator import attrgetter
@@ -262,6 +263,54 @@ def test_supervise(
262263
with expectation:
263264
supervise_task(**kw)
264265

266+
def test_on_kill_hook_called_when_supervisor_receives_sigterm(
267+
self,
268+
test_dags_dir,
269+
captured_logs,
270+
client_with_ti_start,
271+
):
272+
"""SIGTERM to the supervisor process is forwarded to the task subprocess."""
273+
ti = TaskInstanceDTO(
274+
id=uuid7(),
275+
task_id="signal_task",
276+
dag_id="signal_forward_test",
277+
run_id="r",
278+
try_number=1,
279+
dag_version_id=uuid7(),
280+
pool_slots=1,
281+
queue="default",
282+
priority_weight=1,
283+
)
284+
bundle_info = BundleInfo(name="my-bundle", version=None)
285+
286+
supervisor_pid = os.getpid()
287+
288+
def _kill_children():
289+
for child in psutil.Process(supervisor_pid).children(recursive=True):
290+
with suppress(psutil.NoSuchProcess):
291+
child.kill()
292+
293+
watchdog = threading.Timer(20.0, _kill_children)
294+
watchdog.daemon = True
295+
watchdog.start()
296+
297+
try:
298+
with patch.dict(os.environ, local_dag_bundle_cfg(test_dags_dir, bundle_info.name)):
299+
supervise_task(
300+
ti=ti,
301+
dag_rel_path="signal_forward_test.py",
302+
token="",
303+
dry_run=True,
304+
client=client_with_ti_start,
305+
bundle_info=bundle_info,
306+
)
307+
finally:
308+
watchdog.cancel()
309+
310+
stdout_events = [entry["event"] for entry in captured_logs if entry.get("logger") == "task.stdout"]
311+
assert "EXECUTE_STARTED" in stdout_events
312+
assert "ON_KILL_CALLED_VIA_SIGNAL_FORWARDING" in stdout_events
313+
265314

266315
@pytest.mark.usefixtures("disable_capturing")
267316
class TestWatchedSubprocess:

0 commit comments

Comments
 (0)