Skip to content

Commit 7ccd65b

Browse files
pankajastroclaude
andcommitted
Gate watcher task-SDK calls on Airflow 3.1 via shared predicate
The first fix only addressed XCom, but CI showed a second SUPERVISOR_COMMS failure: RuntimeTaskInstance.get_task_states raises NameError on Airflow 3.0 outside a supervisor (dag.test()), so the watcher never observed the producer reaching success and polled to timeout. Replace the per-call-site exception fallback with a single, deterministic gate: - Add airflow_supports_task_sdk_runtime() (AIRFLOW_VERSION >= 3.1.0) in constants. - WatcherTrigger.get_xcom_val uses it: AF >= 3.1 -> task-SDK XCom; AF < 3.1 (incl. 3.0) -> direct-DB ORM (get_xcom_val_af2). Revert get_xcom_val_af3 to the plain SDK call. - build_producer_state_fetcher uses the same predicate (lazy-imported to avoid a plugin-discovery circular import): AF < 3.1 -> direct-DB ORM fetcher; AF >= 3.1 -> SDK get_task_states. On AF 3.0 under dag.test() both XCom and producer-state now read the metadata DB, so the deferred task terminates instead of hanging/polling to timeout. Update existing producer-state SDK-path tests to pin Airflow 3.1.0, and remove the obsolete XCom ImportError-fallback test. Refs apache/airflow#51816, apache/airflow#59093, BOSS-405 Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent 1684182 commit 7ccd65b

5 files changed

Lines changed: 48 additions & 54 deletions

File tree

‎cosmos/constants.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,19 @@
99

1010
AIRFLOW_VERSION = Version(airflow.__version__)
1111

12+
13+
def airflow_supports_task_sdk_runtime(version: Version = AIRFLOW_VERSION) -> bool:
14+
"""Whether Airflow's task-SDK runtime XCom / task-state APIs are usable in the triggerer.
15+
16+
``XCom.get_one`` and ``RuntimeTaskInstance.get_task_states`` import ``SUPERVISOR_COMMS``
17+
from ``airflow.sdk.execution_time.task_runner`` -- a symbol that only exists on Airflow
18+
3.1+. On Airflow 3.0 those APIs raise outside a task-SDK supervisor process (most notably
19+
``dag.test()``, which runs triggers/tasks inline), so callers must read XCom and producer
20+
task state directly from the metadata DB instead. See apache/airflow#51816 and #59093.
21+
"""
22+
return version >= Version("3.1.0")
23+
24+
1225
# dbt materialization for models that are inlined as a CTE into downstream models and never written to the warehouse.
1326
DBT_EPHEMERAL_MATERIALIZATION = "ephemeral"
1427

‎cosmos/operators/_watcher/state.py‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -147,14 +147,21 @@ def build_producer_state_fetcher(
147147
) -> ProducerStateFetcher | None:
148148
"""Return a callable that fetches the producer task state for the given Airflow version."""
149149

150-
if airflow_version < Version("3.0.0"):
150+
# Imported lazily: a module-level import of cosmos.constants here triggers a circular import
151+
# during Airflow plugin discovery (cosmos.plugin imports from a half-initialized constants).
152+
from cosmos.constants import airflow_supports_task_sdk_runtime
153+
154+
# Airflow < 3.1 reads producer task state directly from the metadata DB. The task-SDK
155+
# ``RuntimeTaskInstance.get_task_states`` raises on Airflow 3.0 outside a supervisor
156+
# (e.g. dag.test()), so it is only usable on 3.1+. See apache/airflow#51816, #59093.
157+
if not airflow_supports_task_sdk_runtime(airflow_version):
151158
try:
152159
TaskInstance, create_session = _load_airflow2_dependencies()
153160
except ImportError as exc: # pragma: no cover - defensive guard for stripped test envs
154-
logger.warning("Could not import Airflow 2 state dependencies: %s", exc)
161+
logger.warning("Could not import ORM state dependencies: %s", exc)
155162
return None
156163

157-
def fetch_state_airflow2() -> str | None:
164+
def fetch_state_orm() -> str | None:
158165
with create_session() as session:
159166
ti = (
160167
session.query(TaskInstance)
@@ -169,7 +176,7 @@ def fetch_state_airflow2() -> str | None:
169176
return str(ti.state)
170177
return None
171178

172-
return fetch_state_airflow2
179+
return fetch_state_orm
173180

174181
try:
175182
RuntimeTaskInstance = _load_airflow3_dependencies()

‎cosmos/operators/_watcher/triggerer.py‎

Lines changed: 13 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,8 @@
77

88
from airflow.triggers.base import BaseTrigger, TriggerEvent
99
from asgiref.sync import sync_to_async
10-
from packaging.version import Version
1110

12-
from cosmos.constants import _DBT_STARTUP_EVENTS_XCOM_KEY, AIRFLOW_VERSION
11+
from cosmos.constants import _DBT_STARTUP_EVENTS_XCOM_KEY, AIRFLOW_VERSION, airflow_supports_task_sdk_runtime
1312
from cosmos.listeners.dag_run_listener import EventStatus
1413
from cosmos.log import get_logger
1514
from cosmos.operators._watcher.state import (
@@ -79,25 +78,13 @@ def serialize(self) -> tuple[str, dict[str, Any]]:
7978
async def get_xcom_val_af3(self, key: str) -> Any | None:
8079
from airflow.sdk.execution_time.xcom import XCom
8180

82-
try:
83-
return await sync_to_async(XCom.get_one)(
84-
run_id=self.run_id,
85-
key=key,
86-
task_id=self.producer_task_id,
87-
dag_id=self.dag_id,
88-
map_index=self.map_index,
89-
)
90-
except ImportError:
91-
# On Airflow 3.0, ``XCom.get_one`` unconditionally imports ``SUPERVISOR_COMMS``
92-
# from ``airflow.sdk.execution_time.task_runner`` -- a symbol that only exists on
93-
# Airflow 3.1+. Outside a task-SDK supervisor process (most notably ``dag.test()``,
94-
# which runs deferred triggers inline) that import raises, the deferred task is
95-
# re-queued forever, and the DAG run hangs silently. A real triggerer process has the
96-
# supervisor environment, so the SDK path works there and stays primary; only the
97-
# non-supervisor case falls back to the direct-DB ORM path, which the ``ti.xcom_pull``
98-
# implementation on AF 3.0 reads straight from ``XComModel`` (no supervisor needed).
99-
# See apache/airflow#51816 and apache/airflow#59093.
100-
return await self.get_xcom_val_af2(key)
81+
return await sync_to_async(XCom.get_one)(
82+
run_id=self.run_id,
83+
key=key,
84+
task_id=self.producer_task_id,
85+
dag_id=self.dag_id,
86+
map_index=self.map_index,
87+
)
10188

10289
async def get_xcom_val_af2(self, key: str) -> Any | None:
10390
from airflow.models import TaskInstance
@@ -129,10 +116,12 @@ async def get_xcom_val(self, key: str) -> Any | None:
129116
self.run_id,
130117
self.map_index,
131118
)
132-
if AIRFLOW_VERSION < Version("3.0.0"):
133-
return await self.get_xcom_val_af2(key)
134-
else:
119+
# Airflow 3.0's task-SDK XCom API raises outside a supervisor (e.g. dag.test()), so only
120+
# use it on 3.1+; Airflow < 3.1 reads XCom directly from the metadata DB. See #51816.
121+
if airflow_supports_task_sdk_runtime(AIRFLOW_VERSION):
135122
return await self.get_xcom_val_af3(key)
123+
else:
124+
return await self.get_xcom_val_af2(key)
136125

137126
async def _get_node_status(self) -> Any | None:
138127
"""Return the dbt node status from XCom.

‎tests/operators/_watcher/test_triggerer.py‎

Lines changed: 7 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -52,24 +52,6 @@ async def test_get_xcom_val_af3(self):
5252
)
5353
assert result == expected_value
5454

55-
@pytest.mark.skipif(AIRFLOW_VERSION < Version("3.0.0"), reason="Require Airflow >= 3.0.0")
56-
@pytest.mark.asyncio
57-
async def test_get_xcom_val_af3_falls_back_to_af2_on_import_error(self):
58-
# On Airflow 3.0, XCom.get_one imports SUPERVISOR_COMMS (3.1+ only) and raises
59-
# ImportError outside a supervisor (e.g. dag.test()). The SDK path must fall back to
60-
# the direct-DB ORM path instead of propagating the error. See apache/airflow#51816.
61-
expected_value = {"foo": "bar"}
62-
63-
with patch("cosmos.operators._watcher.triggerer.sync_to_async") as mock_sync_to_async:
64-
mock_get_one = AsyncMock(side_effect=ImportError("cannot import name 'SUPERVISOR_COMMS'"))
65-
mock_sync_to_async.return_value = mock_get_one
66-
67-
with patch.object(self.trigger, "get_xcom_val_af2", AsyncMock(return_value=expected_value)) as mock_af2:
68-
result = await self.trigger.get_xcom_val_af3("test_key")
69-
70-
mock_af2.assert_awaited_once_with("test_key")
71-
assert result == expected_value
72-
7355
@pytest.mark.skipif(AIRFLOW_VERSION >= Version("3.0.0"), reason="Require Airflow < 3.0.0")
7456
@pytest.mark.asyncio
7557
async def test_get_xcom_val_af2(self):
@@ -151,8 +133,8 @@ async def test_parse_dbt_node_status_for_test_sensor(self):
151133
@pytest.mark.parametrize(
152134
"airflow_version, expected_val",
153135
[
154-
(Version("2.11.0"), "af2"), # Airflow < 3 uses get_xcom_val_af2
155-
(Version("3.0.0"), "af3"), # Airflow >= 3 uses get_xcom_val_af3
136+
(Version("2.11.0"), "af2"), # Airflow < 3.1 uses the direct-DB get_xcom_val_af2
137+
(Version("3.1.0"), "af3"), # Airflow >= 3.1 uses the task-SDK get_xcom_val_af3
156138
],
157139
)
158140
async def test_get_xcom_val_branches(self, airflow_version, expected_val):
@@ -287,8 +269,11 @@ def _import_side_effect(name: str, *args, **kwargs):
287269
raise ModuleNotFoundError("missing runtime")
288270
return _real_import(name, *args, **kwargs)
289271

290-
with patch("builtins.__import__", side_effect=_import_side_effect):
291-
state = await self.trigger._get_producer_task_status()
272+
# Pin to 3.1.0 so the task-SDK branch is taken; that is where the RuntimeTaskInstance
273+
# import is attempted (and here blocked), which must degrade to None rather than raise.
274+
with patch("cosmos.operators._watcher.triggerer.AIRFLOW_VERSION", Version("3.1.0")):
275+
with patch("builtins.__import__", side_effect=_import_side_effect):
276+
state = await self.trigger._get_producer_task_status()
292277

293278
assert state is None
294279

‎tests/operators/test_watcher.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1064,8 +1064,8 @@ def test_get_producer_task_status_airflow2_missing_instance(self):
10641064
fetcher.assert_called_once_with()
10651065
assert status is None
10661066

1067-
@pytest.mark.skipif(AIRFLOW_VERSION < Version("3.0.0"), reason="Database lookup path in Airflow < 3.0")
1068-
@patch("cosmos.operators._watcher.base.AIRFLOW_VERSION", new=Version("3.0.0"))
1067+
@pytest.mark.skipif(AIRFLOW_VERSION < Version("3.1.0"), reason="task-SDK get_task_states path is Airflow >= 3.1")
1068+
@patch("cosmos.operators._watcher.base.AIRFLOW_VERSION", new=Version("3.1.0"))
10691069
@patch("airflow.sdk.execution_time.task_runner.RuntimeTaskInstance.get_task_states")
10701070
def test_get_producer_task_status_airflow3(self, mock_get_task_states):
10711071
sensor = self.make_sensor()
@@ -1085,8 +1085,8 @@ def test_get_producer_task_status_airflow3(self, mock_get_task_states):
10851085
dag_id="example_dag", task_ids=[sensor.producer_task_id], run_ids=["run_3"]
10861086
)
10871087

1088-
@pytest.mark.skipif(AIRFLOW_VERSION < Version("3.0.0"), reason="Database lookup path in Airflow < 3.0")
1089-
@patch("cosmos.operators._watcher.base.AIRFLOW_VERSION", new=Version("3.0.0"))
1088+
@pytest.mark.skipif(AIRFLOW_VERSION < Version("3.1.0"), reason="task-SDK get_task_states path is Airflow >= 3.1")
1089+
@patch("cosmos.operators._watcher.base.AIRFLOW_VERSION", new=Version("3.1.0"))
10901090
@patch("airflow.sdk.execution_time.task_runner.RuntimeTaskInstance.get_task_states")
10911091
def test_get_producer_task_status_airflow3_missing_state(self, mock_get_task_states):
10921092
sensor = self.make_sensor()

0 commit comments

Comments
 (0)