Skip to content

Commit 48279b4

Browse files
Fail deferrable KPO when a GC'd pod takes its XCom with it
A deferrable KubernetesPodOperator with do_xcom_push=True was silently marked SUCCESS when its pod was garbage-collected between the trigger firing and task re-entry: the 404 handler in trigger_reentry() returned None for any "success" event, so no return_value XCom was ever pushed and downstream tasks failed unrecoverably at templating time. Keep the existing silent-success behavior when do_xcom_push is off (only pod logs are lost there). When XCom was expected, raise PodNotFoundException so the task fails/retries and recreates the pod along with its XCom sidecar. Fixes: apache#73117 Signed-off-by: Rakesh Ramakrishna Pai <[email protected]>
1 parent c4a0a88 commit 48279b4

2 files changed

Lines changed: 35 additions & 5 deletions

File tree

  • providers/cncf/kubernetes
    • src/airflow/providers/cncf/kubernetes/operators
    • tests/unit/cncf/kubernetes/operators

‎providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/pod.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1144,10 +1144,13 @@ def trigger_reentry(self, context: Context, event: dict[str, Any]) -> Any:
11441144
pod_namespace,
11451145
pod_name,
11461146
)
1147-
if event["status"] == "success":
1148-
# Trigger already observed the pod completed successfully;
1149-
# logs/XCom are unrecoverable but the task itself succeeded.
1147+
if event["status"] == "success" and not self.do_xcom_push:
1148+
# Trigger already observed the pod completed successfully and no
1149+
# XCom was expected; logs are unrecoverable but the task itself succeeded.
11501150
return
1151+
# The pod (and its XCom sidecar) is gone. When XCom was expected, silently
1152+
# succeeding would mark the task SUCCESS with no return_value XCom, breaking
1153+
# downstream tasks. Fail instead so retries recreate the pod and its result.
11511154
raise PodNotFoundException(
11521155
f"Pod {pod_namespace}/{pod_name} not found after resuming from deferral"
11531156
) from e

‎providers/cncf/kubernetes/tests/unit/cncf/kubernetes/operators/test_pod.py‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4093,9 +4093,9 @@ def test_skip_deferral_on_terminated_pod(
40934093
@patch(POD_MANAGER_CLASS)
40944094
@patch(HOOK_CLASS)
40954095
def test_async_trigger_reentry_returns_when_pod_gcd_on_success(self, mocked_hook, mock_manager):
4096-
"""Pod GC'd between trigger firing and reentry should not fail a successful task."""
4096+
"""Pod GC'd between trigger firing and reentry should not fail a successful task (no XCom)."""
40974097
mocked_hook.return_value.get_pod.side_effect = ApiException(status=404, reason="Not Found")
4098-
k = KubernetesPodOperator(task_id="task", deferrable=True)
4098+
k = KubernetesPodOperator(task_id="task", deferrable=True, do_xcom_push=False)
40994099
context = create_context(k)
41004100
context["ti"] = MagicMock()
41014101

@@ -4114,6 +4114,33 @@ def test_async_trigger_reentry_returns_when_pod_gcd_on_success(self, mocked_hook
41144114
# await_pod_completion in _clean must not be called with self.pod=None
41154115
mock_manager.return_value.await_pod_completion.assert_not_called()
41164116

4117+
@patch(POD_MANAGER_CLASS)
4118+
@patch(HOOK_CLASS)
4119+
def test_async_trigger_reentry_raises_pod_not_found_on_success_when_xcom_push(
4120+
self, mocked_hook, mock_manager
4121+
):
4122+
"""Pod GC'd before reentry with do_xcom_push=True must not silently succeed (issue #73117).
4123+
4124+
The XCom sidecar died with the pod, so returning success would leave the task
4125+
SUCCESS with no return_value XCom, unrecoverably breaking downstream tasks.
4126+
Raising lets the task fail/retry so a fresh pod reproduces the result.
4127+
"""
4128+
mocked_hook.return_value.get_pod.side_effect = ApiException(status=404, reason="Not Found")
4129+
k = KubernetesPodOperator(task_id="task", deferrable=True, do_xcom_push=True)
4130+
context = create_context(k)
4131+
context["ti"] = MagicMock()
4132+
4133+
with pytest.raises(PodNotFoundException, match="not found after resuming from deferral"):
4134+
k.trigger_reentry(
4135+
context=context,
4136+
event={
4137+
"status": "success",
4138+
"message": TEST_SUCCESS_MESSAGE,
4139+
"name": TEST_NAME,
4140+
"namespace": TEST_NAMESPACE,
4141+
},
4142+
)
4143+
41174144
@patch(POD_MANAGER_CLASS)
41184145
@patch(HOOK_CLASS)
41194146
def test_async_trigger_reentry_raises_pod_not_found_on_failure(self, mocked_hook, mock_manager):

0 commit comments

Comments
 (0)