Skip to content

Commit b9a3acc

Browse files
committed
Fix deferrable KPO trigger_reentry crash when pod is GC'd before re-entry
When KubernetesPodOperator runs in deferrable mode and the pod is reclaimed by Kubernetes between the trigger firing and the worker re-entering the task, the unguarded `self.pod = self.hook.get_pod(...)` in `trigger_reentry` raises `ApiException(404)` and escapes. The dead-code `if not self.pod:` branch intended to translate this to `PodNotFoundException` is never reached because `get_pod` raises rather than returning `None`. Wrap the `get_pod` call so that: - Non-404 ApiExceptions re-raise unchanged. - 404 + event status "success" returns cleanly (the trigger already observed the pod completed successfully; logs/XCom are unrecoverable but the task itself succeeded). - 404 + non-success event raises `PodNotFoundException` (matches existing dead-code intent). Refs apache#66715.
1 parent c0fba18 commit b9a3acc

2 files changed

Lines changed: 99 additions & 2 deletions

File tree

  • providers/cncf/kubernetes
    • src/airflow/providers/cncf/kubernetes/operators
    • tests/unit/cncf/kubernetes/operators

‎providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/pod.py‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -966,9 +966,30 @@ def trigger_reentry(self, context: Context, event: dict[str, Any]) -> Any:
966966
pod_name = event["name"]
967967
pod_namespace = event["namespace"]
968968

969-
self.pod = self.hook.get_pod(pod_name, pod_namespace)
969+
try:
970+
self.pod = self.hook.get_pod(pod_name, pod_namespace)
971+
except ApiException as e:
972+
if e.status != 404:
973+
raise
974+
# Pod was GC'd between trigger firing and re-entry. This is common
975+
# when the cluster reclaims completed pods aggressively or a
976+
# higher-priority workload (e.g. daemonset) preempts the node.
977+
self.log.warning(
978+
"Pod %s/%s not found after resuming from deferral — already GC'd.",
979+
pod_namespace,
980+
pod_name,
981+
)
982+
if event["status"] == "success":
983+
# Trigger already observed the pod completed successfully;
984+
# logs/XCom are unrecoverable but the task itself succeeded.
985+
return
986+
raise PodNotFoundException(
987+
f"Pod {pod_namespace}/{pod_name} not found after resuming from deferral"
988+
) from e
970989

971990
if not self.pod:
991+
# Defensive: get_pod is documented to raise on missing pods, but
992+
# keep this for any subclass override that returns None instead.
972993
raise PodNotFoundException("Could not find pod after resuming from deferral")
973994

974995
follow = self.logging_interval is None

‎providers/cncf/kubernetes/tests/unit/cncf/kubernetes/operators/test_pod.py‎

Lines changed: 77 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,12 @@
3939
)
4040
from airflow.providers.cncf.kubernetes.secret import Secret
4141
from airflow.providers.cncf.kubernetes.triggers.pod import KubernetesPodTrigger
42-
from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction, PodLoggingStatus, PodPhase
42+
from airflow.providers.cncf.kubernetes.utils.pod_manager import (
43+
OnFinishAction,
44+
PodLoggingStatus,
45+
PodNotFoundException,
46+
PodPhase,
47+
)
4348
from airflow.providers.cncf.kubernetes.utils.xcom_sidecar import PodDefaults
4449
from airflow.providers.common.compat.sdk import (
4550
XCOM_RETURN_KEY,
@@ -3346,6 +3351,77 @@ def test_skip_deferral_on_terminated_pod(
33463351
k.execute(context)
33473352
mocked_trigger_reentry.assert_called_once()
33483353

3354+
@patch(POD_MANAGER_CLASS)
3355+
@patch(HOOK_CLASS)
3356+
def test_async_trigger_reentry_returns_when_pod_gcd_on_success(
3357+
self, mocked_hook, mock_manager
3358+
):
3359+
"""Pod GC'd between trigger firing and reentry should not fail a successful task."""
3360+
mocked_hook.return_value.get_pod.side_effect = ApiException(status=404, reason="Not Found")
3361+
k = KubernetesPodOperator(task_id="task", deferrable=True)
3362+
context = create_context(k)
3363+
context["ti"] = MagicMock()
3364+
3365+
# Should return cleanly without raising
3366+
result = k.trigger_reentry(
3367+
context=context,
3368+
event={
3369+
"status": "success",
3370+
"message": TEST_SUCCESS_MESSAGE,
3371+
"name": TEST_NAME,
3372+
"namespace": TEST_NAMESPACE,
3373+
},
3374+
)
3375+
3376+
assert result is None
3377+
# await_pod_completion in _clean must not be called with self.pod=None
3378+
mock_manager.return_value.await_pod_completion.assert_not_called()
3379+
3380+
@patch(POD_MANAGER_CLASS)
3381+
@patch(HOOK_CLASS)
3382+
def test_async_trigger_reentry_raises_pod_not_found_on_failure(
3383+
self, mocked_hook, mock_manager
3384+
):
3385+
"""Pod GC'd before reentry on a failed event surfaces as PodNotFoundException."""
3386+
mocked_hook.return_value.get_pod.side_effect = ApiException(status=404, reason="Not Found")
3387+
k = KubernetesPodOperator(task_id="task", deferrable=True)
3388+
context = create_context(k)
3389+
context["ti"] = MagicMock()
3390+
3391+
with pytest.raises(PodNotFoundException):
3392+
k.trigger_reentry(
3393+
context=context,
3394+
event={
3395+
"status": "failed",
3396+
"message": "Some failure",
3397+
"name": TEST_NAME,
3398+
"namespace": TEST_NAMESPACE,
3399+
},
3400+
)
3401+
3402+
@patch(POD_MANAGER_CLASS)
3403+
@patch(HOOK_CLASS)
3404+
def test_async_trigger_reentry_propagates_non_404_api_exception(
3405+
self, mocked_hook, mock_manager
3406+
):
3407+
"""Non-404 ApiException from get_pod should propagate unchanged."""
3408+
mocked_hook.return_value.get_pod.side_effect = ApiException(status=500, reason="Server Error")
3409+
k = KubernetesPodOperator(task_id="task", deferrable=True)
3410+
context = create_context(k)
3411+
context["ti"] = MagicMock()
3412+
3413+
with pytest.raises(ApiException) as exc_info:
3414+
k.trigger_reentry(
3415+
context=context,
3416+
event={
3417+
"status": "success",
3418+
"message": TEST_SUCCESS_MESSAGE,
3419+
"name": TEST_NAME,
3420+
"namespace": TEST_NAMESPACE,
3421+
},
3422+
)
3423+
assert exc_info.value.status == 500
3424+
33493425

33503426
@pytest.mark.parametrize("do_xcom_push", [True, False])
33513427
@patch(KUB_OP_PATH.format("extract_xcom"))

0 commit comments

Comments
 (0)