Skip to content

Commit 21499c6

Browse files
committed
Fix deferrable KPO trigger_reentry crash when pod is GC'd before re-entry
When KubernetesPodOperator runs in deferrable mode and the pod is reclaimed by Kubernetes between the trigger firing and the worker re-entering the task, the unguarded `self.pod = self.hook.get_pod(...)` in `trigger_reentry` raises `ApiException(404)` and escapes. The dead-code `if not self.pod:` branch intended to translate this to `PodNotFoundException` is never reached because `get_pod` raises rather than returning `None`. Wrap the `get_pod` call so that: - Non-404 ApiExceptions re-raise unchanged. - 404 + event status "success" returns cleanly (the trigger already observed the pod completed successfully; logs/XCom are unrecoverable but the task itself succeeded). - 404 + non-success event raises `PodNotFoundException` (matches existing dead-code intent). Refs apache#66715.
1 parent bd2c948 commit 21499c6

2 files changed

Lines changed: 99 additions & 2 deletions

File tree

  • providers/cncf/kubernetes
    • src/airflow/providers/cncf/kubernetes/operators
    • tests/unit/cncf/kubernetes/operators

‎providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/pod.py‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -962,9 +962,30 @@ def trigger_reentry(self, context: Context, event: dict[str, Any]) -> Any:
962962
pod_name = event["name"]
963963
pod_namespace = event["namespace"]
964964

965-
self.pod = self.hook.get_pod(pod_name, pod_namespace)
965+
try:
966+
self.pod = self.hook.get_pod(pod_name, pod_namespace)
967+
except ApiException as e:
968+
if e.status != 404:
969+
raise
970+
# Pod was GC'd between trigger firing and re-entry. This is common
971+
# when the cluster reclaims completed pods aggressively or a
972+
# higher-priority workload (e.g. daemonset) preempts the node.
973+
self.log.warning(
974+
"Pod %s/%s not found after resuming from deferral — already GC'd.",
975+
pod_namespace,
976+
pod_name,
977+
)
978+
if event["status"] == "success":
979+
# Trigger already observed the pod completed successfully;
980+
# logs/XCom are unrecoverable but the task itself succeeded.
981+
return
982+
raise PodNotFoundException(
983+
f"Pod {pod_namespace}/{pod_name} not found after resuming from deferral"
984+
) from e
966985

967986
if not self.pod:
987+
# Defensive: get_pod is documented to raise on missing pods, but
988+
# keep this for any subclass override that returns None instead.
968989
raise PodNotFoundException("Could not find pod after resuming from deferral")
969990

970991
follow = self.logging_interval is None

‎providers/cncf/kubernetes/tests/unit/cncf/kubernetes/operators/test_pod.py‎

Lines changed: 77 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,12 @@
3939
)
4040
from airflow.providers.cncf.kubernetes.secret import Secret
4141
from airflow.providers.cncf.kubernetes.triggers.pod import KubernetesPodTrigger
42-
from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction, PodLoggingStatus, PodPhase
42+
from airflow.providers.cncf.kubernetes.utils.pod_manager import (
43+
OnFinishAction,
44+
PodLoggingStatus,
45+
PodNotFoundException,
46+
PodPhase,
47+
)
4348
from airflow.providers.cncf.kubernetes.utils.xcom_sidecar import PodDefaults
4449
from airflow.providers.common.compat.sdk import (
4550
XCOM_RETURN_KEY,
@@ -3123,6 +3128,77 @@ def test_skip_deferral_on_terminated_pod(
31233128
k.execute(context)
31243129
mocked_trigger_reentry.assert_called_once()
31253130

3131+
@patch(POD_MANAGER_CLASS)
3132+
@patch(HOOK_CLASS)
3133+
def test_async_trigger_reentry_returns_when_pod_gcd_on_success(
3134+
self, mocked_hook, mock_manager
3135+
):
3136+
"""Pod GC'd between trigger firing and reentry should not fail a successful task."""
3137+
mocked_hook.return_value.get_pod.side_effect = ApiException(status=404, reason="Not Found")
3138+
k = KubernetesPodOperator(task_id="task", deferrable=True)
3139+
context = create_context(k)
3140+
context["ti"] = MagicMock()
3141+
3142+
# Should return cleanly without raising
3143+
result = k.trigger_reentry(
3144+
context=context,
3145+
event={
3146+
"status": "success",
3147+
"message": TEST_SUCCESS_MESSAGE,
3148+
"name": TEST_NAME,
3149+
"namespace": TEST_NAMESPACE,
3150+
},
3151+
)
3152+
3153+
assert result is None
3154+
# await_pod_completion in _clean must not be called with self.pod=None
3155+
mock_manager.return_value.await_pod_completion.assert_not_called()
3156+
3157+
@patch(POD_MANAGER_CLASS)
3158+
@patch(HOOK_CLASS)
3159+
def test_async_trigger_reentry_raises_pod_not_found_on_failure(
3160+
self, mocked_hook, mock_manager
3161+
):
3162+
"""Pod GC'd before reentry on a failed event surfaces as PodNotFoundException."""
3163+
mocked_hook.return_value.get_pod.side_effect = ApiException(status=404, reason="Not Found")
3164+
k = KubernetesPodOperator(task_id="task", deferrable=True)
3165+
context = create_context(k)
3166+
context["ti"] = MagicMock()
3167+
3168+
with pytest.raises(PodNotFoundException):
3169+
k.trigger_reentry(
3170+
context=context,
3171+
event={
3172+
"status": "failed",
3173+
"message": "Some failure",
3174+
"name": TEST_NAME,
3175+
"namespace": TEST_NAMESPACE,
3176+
},
3177+
)
3178+
3179+
@patch(POD_MANAGER_CLASS)
3180+
@patch(HOOK_CLASS)
3181+
def test_async_trigger_reentry_propagates_non_404_api_exception(
3182+
self, mocked_hook, mock_manager
3183+
):
3184+
"""Non-404 ApiException from get_pod should propagate unchanged."""
3185+
mocked_hook.return_value.get_pod.side_effect = ApiException(status=500, reason="Server Error")
3186+
k = KubernetesPodOperator(task_id="task", deferrable=True)
3187+
context = create_context(k)
3188+
context["ti"] = MagicMock()
3189+
3190+
with pytest.raises(ApiException) as exc_info:
3191+
k.trigger_reentry(
3192+
context=context,
3193+
event={
3194+
"status": "success",
3195+
"message": TEST_SUCCESS_MESSAGE,
3196+
"name": TEST_NAME,
3197+
"namespace": TEST_NAMESPACE,
3198+
},
3199+
)
3200+
assert exc_info.value.status == 500
3201+
31263202

31273203
@pytest.mark.parametrize("do_xcom_push", [True, False])
31283204
@patch(KUB_OP_PATH.format("extract_xcom"))

0 commit comments

Comments
 (0)