Skip to content

Commit bda472d

Browse files
authored
Fix deferrable KPO trigger_reentry crash when pod is GC'd before re-entry (#66716)
* Fix deferrable KPO trigger_reentry crash when pod is GC'd before re-entry When KubernetesPodOperator runs in deferrable mode and the pod is reclaimed by Kubernetes between the trigger firing and the worker re-entering the task, the unguarded `self.pod = self.hook.get_pod(...)` in `trigger_reentry` raises `ApiException(404)` and escapes. The dead-code `if not self.pod:` branch intended to translate this to `PodNotFoundException` is never reached because `get_pod` raises rather than returning `None`. Wrap the `get_pod` call so that: - Non-404 ApiExceptions re-raise unchanged. - 404 + event status "success" returns cleanly (the trigger already observed the pod completed successfully; logs/XCom are unrecoverable but the task itself succeeded). - 404 + non-success event raises `PodNotFoundException` (matches existing dead-code intent). Refs #66715. * fix: apply ruff-format to new test signatures
1 parent 41e16d5 commit bda472d

2 files changed

Lines changed: 93 additions & 2 deletions

File tree

  • providers/cncf/kubernetes
    • src/airflow/providers/cncf/kubernetes/operators
    • tests/unit/cncf/kubernetes/operators

‎providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/pod.py‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -966,9 +966,30 @@ def trigger_reentry(self, context: Context, event: dict[str, Any]) -> Any:
966966
pod_name = event["name"]
967967
pod_namespace = event["namespace"]
968968

969-
self.pod = self.hook.get_pod(pod_name, pod_namespace)
969+
try:
970+
self.pod = self.hook.get_pod(pod_name, pod_namespace)
971+
except ApiException as e:
972+
if e.status != 404:
973+
raise
974+
# Pod was GC'd between trigger firing and re-entry. This is common
975+
# when the cluster reclaims completed pods aggressively or a
976+
# higher-priority workload (e.g. daemonset) preempts the node.
977+
self.log.warning(
978+
"Pod %s/%s not found after resuming from deferral — already GC'd.",
979+
pod_namespace,
980+
pod_name,
981+
)
982+
if event["status"] == "success":
983+
# Trigger already observed the pod completed successfully;
984+
# logs/XCom are unrecoverable but the task itself succeeded.
985+
return
986+
raise PodNotFoundException(
987+
f"Pod {pod_namespace}/{pod_name} not found after resuming from deferral"
988+
) from e
970989

971990
if not self.pod:
991+
# Defensive: get_pod is documented to raise on missing pods, but
992+
# keep this for any subclass override that returns None instead.
972993
raise PodNotFoundException("Could not find pod after resuming from deferral")
973994

974995
follow = self.logging_interval is None

‎providers/cncf/kubernetes/tests/unit/cncf/kubernetes/operators/test_pod.py‎

Lines changed: 71 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,12 @@
3939
)
4040
from airflow.providers.cncf.kubernetes.secret import Secret
4141
from airflow.providers.cncf.kubernetes.triggers.pod import KubernetesPodTrigger
42-
from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction, PodLoggingStatus, PodPhase
42+
from airflow.providers.cncf.kubernetes.utils.pod_manager import (
43+
OnFinishAction,
44+
PodLoggingStatus,
45+
PodNotFoundException,
46+
PodPhase,
47+
)
4348
from airflow.providers.cncf.kubernetes.utils.xcom_sidecar import PodDefaults
4449
from airflow.providers.common.compat.sdk import (
4550
XCOM_RETURN_KEY,
@@ -3346,6 +3351,71 @@ def test_skip_deferral_on_terminated_pod(
33463351
k.execute(context)
33473352
mocked_trigger_reentry.assert_called_once()
33483353

3354+
@patch(POD_MANAGER_CLASS)
3355+
@patch(HOOK_CLASS)
3356+
def test_async_trigger_reentry_returns_when_pod_gcd_on_success(self, mocked_hook, mock_manager):
3357+
"""Pod GC'd between trigger firing and reentry should not fail a successful task."""
3358+
mocked_hook.return_value.get_pod.side_effect = ApiException(status=404, reason="Not Found")
3359+
k = KubernetesPodOperator(task_id="task", deferrable=True)
3360+
context = create_context(k)
3361+
context["ti"] = MagicMock()
3362+
3363+
# Should return cleanly without raising
3364+
result = k.trigger_reentry(
3365+
context=context,
3366+
event={
3367+
"status": "success",
3368+
"message": TEST_SUCCESS_MESSAGE,
3369+
"name": TEST_NAME,
3370+
"namespace": TEST_NAMESPACE,
3371+
},
3372+
)
3373+
3374+
assert result is None
3375+
# await_pod_completion in _clean must not be called with self.pod=None
3376+
mock_manager.return_value.await_pod_completion.assert_not_called()
3377+
3378+
@patch(POD_MANAGER_CLASS)
3379+
@patch(HOOK_CLASS)
3380+
def test_async_trigger_reentry_raises_pod_not_found_on_failure(self, mocked_hook, mock_manager):
3381+
"""Pod GC'd before reentry on a failed event surfaces as PodNotFoundException."""
3382+
mocked_hook.return_value.get_pod.side_effect = ApiException(status=404, reason="Not Found")
3383+
k = KubernetesPodOperator(task_id="task", deferrable=True)
3384+
context = create_context(k)
3385+
context["ti"] = MagicMock()
3386+
3387+
with pytest.raises(PodNotFoundException):
3388+
k.trigger_reentry(
3389+
context=context,
3390+
event={
3391+
"status": "failed",
3392+
"message": "Some failure",
3393+
"name": TEST_NAME,
3394+
"namespace": TEST_NAMESPACE,
3395+
},
3396+
)
3397+
3398+
@patch(POD_MANAGER_CLASS)
3399+
@patch(HOOK_CLASS)
3400+
def test_async_trigger_reentry_propagates_non_404_api_exception(self, mocked_hook, mock_manager):
3401+
"""Non-404 ApiException from get_pod should propagate unchanged."""
3402+
mocked_hook.return_value.get_pod.side_effect = ApiException(status=500, reason="Server Error")
3403+
k = KubernetesPodOperator(task_id="task", deferrable=True)
3404+
context = create_context(k)
3405+
context["ti"] = MagicMock()
3406+
3407+
with pytest.raises(ApiException) as exc_info:
3408+
k.trigger_reentry(
3409+
context=context,
3410+
event={
3411+
"status": "success",
3412+
"message": TEST_SUCCESS_MESSAGE,
3413+
"name": TEST_NAME,
3414+
"namespace": TEST_NAMESPACE,
3415+
},
3416+
)
3417+
assert exc_info.value.status == 500
3418+
33493419

33503420
@pytest.mark.parametrize("do_xcom_push", [True, False])
33513421
@patch(KUB_OP_PATH.format("extract_xcom"))

0 commit comments

Comments
 (0)