Skip to content

Commit 106b91c

Browse files
committed
Forward termination signals from supervisor to task subprocess
When a Kubernetes worker pod receives SIGTERM (e.g. spot interruption, scaling down), the signal is delivered to the supervisor process (PID 1 in the container). Previously, the supervisor had no signal handler and would exit with default behavior, leaving the task subprocess orphaned without ever calling the operator's on_kill() hook. This meant spawned resources (pods, subprocesses, etc.) were never cleaned up. This change installs SIGTERM/SIGINT signal handlers in the supervise() function that forward the received signal to the task subprocess. The child process already has a signal handler (registered in task_runner.py) that calls on_kill() when it receives SIGTERM, so forwarding the signal completes the chain and restores the Airflow 2 behavior. Fixes: apache#58936
1 parent 57b3f2f commit 106b91c

2 files changed

Lines changed: 116 additions & 1 deletion

File tree

‎task-sdk/src/airflow/sdk/execution_time/supervisor.py‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2078,7 +2078,33 @@ def supervise(
20782078
sentry_integration=sentry_integration,
20792079
)
20802080

2081-
exit_code = process.wait()
2081+
# Forward termination signals to the task subprocess so that the operator's
2082+
# on_kill() hook is invoked on graceful shutdown (e.g. K8s pod SIGTERM).
2083+
# Without this, the supervisor exits on SIGTERM without notifying the child,
2084+
# leaving spawned resources (pods, subprocesses, etc.) running.
2085+
prev_sigterm = signal.getsignal(signal.SIGTERM)
2086+
prev_sigint = signal.getsignal(signal.SIGINT)
2087+
2088+
def _forward_signal(signum, frame):
2089+
log.info(
2090+
"Received signal, forwarding to task subprocess",
2091+
signal=signal.Signals(signum).name,
2092+
pid=process.pid,
2093+
)
2094+
try:
2095+
os.kill(process.pid, signum)
2096+
except ProcessLookupError:
2097+
pass
2098+
2099+
signal.signal(signal.SIGTERM, _forward_signal)
2100+
signal.signal(signal.SIGINT, _forward_signal)
2101+
2102+
try:
2103+
exit_code = process.wait()
2104+
finally:
2105+
signal.signal(signal.SIGTERM, prev_sigterm)
2106+
signal.signal(signal.SIGINT, prev_sigint)
2107+
20822108
end = time.monotonic()
20832109
log.info(
20842110
"Task finished",

‎task-sdk/tests/task_sdk/execution_time/test_supervisor.py‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -482,6 +482,95 @@ def on_kill(self) -> None:
482482
captured = capfd.readouterr()
483483
assert "On kill hook called!" in captured.out
484484

485+
def test_on_kill_hook_called_when_supervisor_receives_sigterm(
486+
self,
487+
client_with_ti_start,
488+
mocked_parse,
489+
make_ti_context,
490+
mock_supervisor_comms,
491+
create_runtime_ti,
492+
make_ti_context_dict,
493+
capfd,
494+
):
495+
"""Test that SIGTERM to the supervisor process is forwarded to the task subprocess.
496+
497+
This simulates what happens when Kubernetes sends SIGTERM to the worker pod:
498+
the supervisor should forward the signal to the child process so that the
499+
operator's on_kill() hook is triggered for resource cleanup.
500+
"""
501+
import threading
502+
503+
ti_id = "4d828a62-a417-4936-a7a6-2b3fabacecab"
504+
505+
def handle_request(request: httpx.Request) -> httpx.Response:
506+
if request.url.path == f"/task-instances/{ti_id}/run":
507+
return httpx.Response(200, json=make_ti_context_dict())
508+
return httpx.Response(status_code=204)
509+
510+
def subprocess_main():
511+
CommsDecoder()._get_response()
512+
513+
class CustomOperator(BaseOperator):
514+
def execute(self, context):
515+
for i in range(1000):
516+
print(f"Iteration {i}")
517+
sleep(1)
518+
519+
def on_kill(self) -> None:
520+
print("On kill hook called via signal forwarding!")
521+
raise SystemExit(0)
522+
523+
task = CustomOperator(task_id="test-signal-forward")
524+
runtime_ti = create_runtime_ti(
525+
dag_id="c",
526+
task=task,
527+
conf={},
528+
)
529+
run(runtime_ti, context=runtime_ti.get_template_context(), log=mock.MagicMock())
530+
531+
proc = ActivitySubprocess.start(
532+
dag_rel_path=os.devnull,
533+
bundle_info=FAKE_BUNDLE,
534+
what=TaskInstance(
535+
id=ti_id,
536+
task_id="b",
537+
dag_id="c",
538+
run_id="d",
539+
try_number=1,
540+
dag_version_id=uuid7(),
541+
),
542+
client=make_client(transport=httpx.MockTransport(handle_request)),
543+
target=subprocess_main,
544+
)
545+
546+
# Install signal forwarding handler (same mechanism as supervise() does)
547+
prev_sigterm = signal.getsignal(signal.SIGTERM)
548+
549+
def _forward_signal(signum, frame):
550+
try:
551+
os.kill(proc.pid, signum)
552+
except ProcessLookupError:
553+
pass
554+
555+
signal.signal(signal.SIGTERM, _forward_signal)
556+
557+
# Send SIGTERM to ourselves (the supervisor) from a background thread,
558+
# giving the subprocess time to start executing first.
559+
def send_sigterm_delayed():
560+
sleep(2)
561+
os.kill(os.getpid(), signal.SIGTERM)
562+
563+
t = threading.Thread(target=send_sigterm_delayed, daemon=True)
564+
t.start()
565+
566+
try:
567+
proc.wait()
568+
finally:
569+
signal.signal(signal.SIGTERM, prev_sigterm)
570+
571+
captured = capfd.readouterr()
572+
assert "On kill hook called via signal forwarding!" in captured.out
573+
485574
def test_subprocess_sigkilled(self, client_with_ti_start):
486575
main_pid = os.getpid()
487576

0 commit comments

Comments
 (0)