Skip to content

Commit d3b4a91

Browse files
Lee-Wdstandishjedcunningham
authored
fix: Avoid retrying after KubernetesPodOperator has been marked as failed (#36749)
* fix(providers/cncf): avoid retrying after KubernetesPodOperator has been marked as failed #36471 * feat(providers/cncf): add pod name to on_kill log * refactor(providers/cncf): refine comments and log message as suggested * docs(providers/cncf): reword comment Co-authored-by: Jed Cunningham <[email protected]> * fix(providers/cncf): move the _killed flag to cleanup method if the task has been killed, do not cleanup again --------- Co-authored-by: Daniel Standish <[email protected]> Co-authored-by: Jed Cunningham <[email protected]>
1 parent 09bb1a8 commit d3b4a91

1 file changed

Lines changed: 14 additions & 1 deletion

File tree

  • airflow/providers/cncf/kubernetes/operators

‎airflow/providers/cncf/kubernetes/operators/pod.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
from functools import cached_property
3030
from typing import TYPE_CHECKING, Any, Callable, Iterable, Sequence
3131

32+
import kubernetes
3233
from kubernetes.client import CoreV1Api, V1Pod, models as k8s
3334
from kubernetes.stream import stream
3435
from urllib3.exceptions import HTTPError
@@ -380,6 +381,7 @@ def __init__(
380381

381382
self._config_dict: dict | None = None # TODO: remove it when removing convert_config_file_to_dict
382383
self._progress_callback = progress_callback
384+
self._killed: bool = False
383385

384386
@cached_property
385387
def _incluster_namespace(self):
@@ -577,6 +579,7 @@ def execute_sync(self, context: Context):
577579
pod=self.pod or self.pod_request_obj,
578580
remote_pod=self.remote_pod,
579581
)
582+
580583
if self.do_xcom_push:
581584
return result
582585

@@ -676,6 +679,11 @@ def post_complete_action(self, *, pod, remote_pod, **kwargs):
676679
)
677680

678681
def cleanup(self, pod: k8s.V1Pod, remote_pod: k8s.V1Pod):
682+
# If a task got marked as failed, "on_kill" method would be called and the pod will be cleaned up
683+
# there. Cleaning it up again will raise an exception (which might cause retry).
684+
if self._killed:
685+
return
686+
679687
istio_enabled = self.is_istio_enabled(remote_pod)
680688
pod_phase = remote_pod.status.phase if hasattr(remote_pod, "status") else None
681689

@@ -818,6 +826,7 @@ def patch_already_checked(self, pod: k8s.V1Pod, *, reraise=True):
818826
)
819827

820828
def on_kill(self) -> None:
829+
self._killed = True
821830
if self.pod:
822831
pod = self.pod
823832
kwargs = {
@@ -826,7 +835,11 @@ def on_kill(self) -> None:
826835
}
827836
if self.termination_grace_period is not None:
828837
kwargs.update(grace_period_seconds=self.termination_grace_period)
829-
self.client.delete_namespaced_pod(**kwargs)
838+
839+
try:
840+
self.client.delete_namespaced_pod(**kwargs)
841+
except kubernetes.client.exceptions.ApiException:
842+
self.log.exception("Unable to delete pod %s", self.pod.metadata.name)
830843

831844
def build_pod_request_obj(self, context: Context | None = None) -> k8s.V1Pod:
832845
"""

0 commit comments

Comments
 (0)