Skip to content

Commit 4b780d9

Browse files
committed
Forward termination signals from supervisor to task subprocess
When a Kubernetes worker pod receives SIGTERM (e.g. spot interruption, scaling down), the signal is delivered to the supervisor process (PID 1 in the container). Previously, the supervisor had no signal handler and would exit with default behavior, leaving the task subprocess orphaned without ever calling the operator's on_kill() hook. This meant spawned resources (pods, subprocesses, etc.) were never cleaned up. This change installs SIGTERM/SIGINT signal handlers in the supervise() function that forward the received signal to the task subprocess. The child process already has a signal handler (registered in task_runner.py) that calls on_kill() when it receives SIGTERM, so forwarding the signal completes the chain and restores the Airflow 2 behavior. Fixes: apache#58936 Signed-off-by: André Ahlert <[email protected]>
1 parent cdf00af commit 4b780d9

2 files changed

Lines changed: 116 additions & 1 deletion

File tree

‎task-sdk/src/airflow/sdk/execution_time/supervisor.py‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2232,7 +2232,33 @@ def supervise_task(
22322232
sentry_integration=sentry_integration,
22332233
)
22342234

2235-
exit_code = process.wait()
2235+
# Forward termination signals to the task subprocess so that the operator's
2236+
# on_kill() hook is invoked on graceful shutdown (e.g. K8s pod SIGTERM).
2237+
# Without this, the supervisor exits on SIGTERM without notifying the child,
2238+
# leaving spawned resources (pods, subprocesses, etc.) running.
2239+
prev_sigterm = signal.getsignal(signal.SIGTERM)
2240+
prev_sigint = signal.getsignal(signal.SIGINT)
2241+
2242+
def _forward_signal(signum, frame):
2243+
log.info(
2244+
"Received signal, forwarding to task subprocess",
2245+
signal=signal.Signals(signum).name,
2246+
pid=process.pid,
2247+
)
2248+
try:
2249+
os.kill(process.pid, signum)
2250+
except ProcessLookupError:
2251+
pass
2252+
2253+
signal.signal(signal.SIGTERM, _forward_signal)
2254+
signal.signal(signal.SIGINT, _forward_signal)
2255+
2256+
try:
2257+
exit_code = process.wait()
2258+
finally:
2259+
signal.signal(signal.SIGTERM, prev_sigterm)
2260+
signal.signal(signal.SIGINT, prev_sigint)
2261+
22362262
end = time.monotonic()
22372263
log.info(
22382264
"Workload finished",

‎task-sdk/tests/task_sdk/execution_time/test_supervisor.py‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -488,6 +488,95 @@ def on_kill(self) -> None:
488488
captured = capfd.readouterr()
489489
assert "On kill hook called!" in captured.out
490490

491+
def test_on_kill_hook_called_when_supervisor_receives_sigterm(
492+
self,
493+
client_with_ti_start,
494+
mocked_parse,
495+
make_ti_context,
496+
mock_supervisor_comms,
497+
create_runtime_ti,
498+
make_ti_context_dict,
499+
capfd,
500+
):
501+
"""Test that SIGTERM to the supervisor process is forwarded to the task subprocess.
502+
503+
This simulates what happens when Kubernetes sends SIGTERM to the worker pod:
504+
the supervisor should forward the signal to the child process so that the
505+
operator's on_kill() hook is triggered for resource cleanup.
506+
"""
507+
import threading
508+
509+
ti_id = "4d828a62-a417-4936-a7a6-2b3fabacecab"
510+
511+
def handle_request(request: httpx.Request) -> httpx.Response:
512+
if request.url.path == f"/task-instances/{ti_id}/run":
513+
return httpx.Response(200, json=make_ti_context_dict())
514+
return httpx.Response(status_code=204)
515+
516+
def subprocess_main():
517+
CommsDecoder()._get_response()
518+
519+
class CustomOperator(BaseOperator):
520+
def execute(self, context):
521+
for i in range(1000):
522+
print(f"Iteration {i}")
523+
sleep(1)
524+
525+
def on_kill(self) -> None:
526+
print("On kill hook called via signal forwarding!")
527+
raise SystemExit(0)
528+
529+
task = CustomOperator(task_id="test-signal-forward")
530+
runtime_ti = create_runtime_ti(
531+
dag_id="c",
532+
task=task,
533+
conf={},
534+
)
535+
run(runtime_ti, context=runtime_ti.get_template_context(), log=mock.MagicMock())
536+
537+
proc = ActivitySubprocess.start(
538+
dag_rel_path=os.devnull,
539+
bundle_info=FAKE_BUNDLE,
540+
what=TaskInstance(
541+
id=ti_id,
542+
task_id="b",
543+
dag_id="c",
544+
run_id="d",
545+
try_number=1,
546+
dag_version_id=uuid7(),
547+
),
548+
client=make_client(transport=httpx.MockTransport(handle_request)),
549+
target=subprocess_main,
550+
)
551+
552+
# Install signal forwarding handler (same mechanism as supervise() does)
553+
prev_sigterm = signal.getsignal(signal.SIGTERM)
554+
555+
def _forward_signal(signum, frame):
556+
try:
557+
os.kill(proc.pid, signum)
558+
except ProcessLookupError:
559+
pass
560+
561+
signal.signal(signal.SIGTERM, _forward_signal)
562+
563+
# Send SIGTERM to ourselves (the supervisor) from a background thread,
564+
# giving the subprocess time to start executing first.
565+
def send_sigterm_delayed():
566+
sleep(2)
567+
os.kill(os.getpid(), signal.SIGTERM)
568+
569+
t = threading.Thread(target=send_sigterm_delayed, daemon=True)
570+
t.start()
571+
572+
try:
573+
proc.wait()
574+
finally:
575+
signal.signal(signal.SIGTERM, prev_sigterm)
576+
577+
captured = capfd.readouterr()
578+
assert "On kill hook called via signal forwarding!" in captured.out
579+
491580
def test_subprocess_sigkilled(self, client_with_ti_start):
492581
main_pid = os.getpid()
493582

0 commit comments

Comments
 (0)