Skip to content

Commit 1a2784d

Browse files
authored
Fix ExternalTaskSensor to use timeout parameter in deferrable mode (#… (#62556)
* Fix ExternalTaskSensor to use timeout parameter in deferrable mode (#62516) ExternalTaskSensor was incorrectly using execution_timeout instead of timeout when deferring in deferrable mode, causing inconsistent behavior across poke, reschedule, and deferrable modes. Changes: - Updated execute() to use timedelta(seconds=self.timeout) with fallback to execution_timeout - Added comprehensive test coverage for timeout behavior in deferrable mode - Ensures consistent timeout semantics across all sensor modes * Simplify timeout logic and update tests Remove incorrect fallback to execution_timeout. Timeout parameter always has a default value from config, so no fallback is needed or correct. Update tests to validate correct behavior matching other sensors * Remove empty SOLUTION.md file * Fix ExternalTaskSensor timeout handling in deferrable mode - Implement fallback to execution_timeout when timeout is not set (0 or None) - Update tests to cover 3 distinct scenarios: 1. timeout set, execution_timeout not set 2. timeout not set (0), execution_timeout set 3. timeout and execution_timeout both set (priority test) - Fix whitespace issue flagged by ruff * Retrigger CI
1 parent d4765cf commit 1a2784d

2 files changed

Lines changed: 78 additions & 2 deletions

File tree

‎providers/standard/src/airflow/providers/standard/sensors/external_task.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -432,10 +432,15 @@ def execute(self, context: Context) -> None:
432432
if not self.deferrable:
433433
super().execute(context)
434434
else:
435+
# Determine the timeout to use: prefer timeout parameter, fallback to execution_timeout
436+
timeout_value = self.timeout
437+
if not timeout_value and self.execution_timeout:
438+
timeout_value = self.execution_timeout.total_seconds()
439+
435440
dttm_filter = self._get_dttm_filter(context)
436441
if AIRFLOW_V_3_0_PLUS:
437442
self.defer(
438-
timeout=self.execution_timeout,
443+
timeout=datetime.timedelta(seconds=timeout_value) if timeout_value else None,
439444
trigger=WorkflowTrigger(
440445
external_dag_id=self.external_dag_id,
441446
external_task_group_id=self.external_task_group_id,
@@ -453,7 +458,7 @@ def execute(self, context: Context) -> None:
453458
)
454459
else:
455460
self.defer(
456-
timeout=self.execution_timeout,
461+
timeout=datetime.timedelta(seconds=timeout_value) if timeout_value else None,
457462
trigger=WorkflowTrigger(
458463
external_dag_id=self.external_dag_id,
459464
external_task_group_id=self.external_task_group_id,

‎providers/standard/tests/unit/standard/sensors/test_external_task_sensor.py‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1049,6 +1049,77 @@ def test_external_task_sensor_deferrable(self, dag_maker):
10491049
assert exc.value.trigger.external_task_ids == ["test_task"]
10501050
assert exc.value.trigger.execution_dates == [DEFAULT_DATE]
10511051

1052+
@pytest.mark.execution_timeout(10)
1053+
def test_external_task_sensor_deferrable_timeout_only(self, dag_maker):
1054+
"""Test that deferrable mode uses timeout parameter when only timeout is set."""
1055+
context = {"execution_date": DEFAULT_DATE}
1056+
with dag_maker() as dag:
1057+
op = ExternalTaskSensor(
1058+
task_id="test_external_task_sensor_check",
1059+
external_dag_id="test_dag_parent",
1060+
external_task_id="test_task",
1061+
deferrable=True,
1062+
timeout=60,
1063+
)
1064+
dr = dag.create_dagrun(
1065+
run_id="test_run",
1066+
run_type=DagRunType.MANUAL,
1067+
state=None,
1068+
)
1069+
context.update(dag_run=dr, logical_date=DEFAULT_DATE)
1070+
1071+
with pytest.raises(TaskDeferred) as exc:
1072+
op.execute(context=context)
1073+
assert exc.value.timeout == timedelta(seconds=60)
1074+
1075+
@pytest.mark.execution_timeout(10)
1076+
def test_external_task_sensor_deferrable_execution_timeout_only(self, dag_maker):
1077+
"""Test that deferrable mode falls back to execution_timeout when timeout is not set."""
1078+
context = {"execution_date": DEFAULT_DATE}
1079+
with dag_maker() as dag:
1080+
op = ExternalTaskSensor(
1081+
task_id="test_external_task_sensor_check",
1082+
external_dag_id="test_dag_parent",
1083+
external_task_id="test_task",
1084+
deferrable=True,
1085+
timeout=0, # Explicitly set to 0 to indicate not using timeout
1086+
execution_timeout=timedelta(seconds=120),
1087+
)
1088+
dr = dag.create_dagrun(
1089+
run_id="test_run",
1090+
run_type=DagRunType.MANUAL,
1091+
state=None,
1092+
)
1093+
context.update(dag_run=dr, logical_date=DEFAULT_DATE)
1094+
1095+
with pytest.raises(TaskDeferred) as exc:
1096+
op.execute(context=context)
1097+
assert exc.value.timeout == timedelta(seconds=120)
1098+
1099+
@pytest.mark.execution_timeout(10)
1100+
def test_external_task_sensor_deferrable_timeout_priority(self, dag_maker):
1101+
"""Test that deferrable mode prioritizes timeout over execution_timeout when both are set."""
1102+
context = {"execution_date": DEFAULT_DATE}
1103+
with dag_maker() as dag:
1104+
op = ExternalTaskSensor(
1105+
task_id="test_external_task_sensor_check",
1106+
external_dag_id="test_dag_parent",
1107+
external_task_id="test_task",
1108+
deferrable=True,
1109+
timeout=90,
1110+
execution_timeout=timedelta(seconds=120),
1111+
)
1112+
dr = dag.create_dagrun(
1113+
run_id="test_run",
1114+
run_type=DagRunType.MANUAL,
1115+
state=None,
1116+
)
1117+
context.update(dag_run=dr, logical_date=DEFAULT_DATE)
1118+
1119+
with pytest.raises(TaskDeferred) as exc:
1120+
op.execute(context=context)
1121+
assert exc.value.timeout == timedelta(seconds=90)
1122+
10521123
def test_get_logical_date(self):
10531124
"""For AF 2, we check for execution_date in context."""
10541125
context = {"execution_date": DEFAULT_DATE}

0 commit comments

Comments
 (0)