Skip to content

Commit 1684182

Browse files
pankajastroclaude
andcommitted
Fix WatcherTrigger XCom fetch hanging dag.test() on Airflow 3.0
On Airflow 3.0, XCom.get_one (airflow.sdk) unconditionally imports SUPERVISOR_COMMS, a symbol that only exists on Airflow 3.1+. Outside a task-SDK supervisor process -- most notably dag.test(), which runs deferred triggers inline -- that import raises, the deferred task is re-queued forever, and the DAG run hangs silently. Wrap the SDK XCom.get_one call in get_xcom_val_af3 with try/except ImportError and fall back to the direct-DB ORM path (get_xcom_val_af2). The SDK path stays primary on AF 3 (a real triggerer has SUPERVISOR_COMMS); only the non-supervisor case falls back. On AF 3.0 ti.xcom_pull reads straight from XComModel, so the fallback works without a supervisor. Also drop the watcher_with_freshness_check.py AF-3.0 .airflowignore skip added as a CI workaround in #2692, and add a regression test for the fallback. Refs apache/airflow#51816, apache/airflow#59093, BOSS-405 Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent 9b3a645 commit 1684182

3 files changed

Lines changed: 37 additions & 18 deletions

File tree

‎cosmos/operators/_watcher/triggerer.py‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -79,13 +79,25 @@ def serialize(self) -> tuple[str, dict[str, Any]]:
7979
async def get_xcom_val_af3(self, key: str) -> Any | None:
8080
from airflow.sdk.execution_time.xcom import XCom
8181

82-
return await sync_to_async(XCom.get_one)(
83-
run_id=self.run_id,
84-
key=key,
85-
task_id=self.producer_task_id,
86-
dag_id=self.dag_id,
87-
map_index=self.map_index,
88-
)
82+
try:
83+
return await sync_to_async(XCom.get_one)(
84+
run_id=self.run_id,
85+
key=key,
86+
task_id=self.producer_task_id,
87+
dag_id=self.dag_id,
88+
map_index=self.map_index,
89+
)
90+
except ImportError:
91+
# On Airflow 3.0, ``XCom.get_one`` unconditionally imports ``SUPERVISOR_COMMS``
92+
# from ``airflow.sdk.execution_time.task_runner`` -- a symbol that only exists on
93+
# Airflow 3.1+. Outside a task-SDK supervisor process (most notably ``dag.test()``,
94+
# which runs deferred triggers inline) that import raises, the deferred task is
95+
# re-queued forever, and the DAG run hangs silently. A real triggerer process has the
96+
# supervisor environment, so the SDK path works there and stays primary; only the
97+
# non-supervisor case falls back to the direct-DB ORM path, which the ``ti.xcom_pull``
98+
# implementation on AF 3.0 reads straight from ``XComModel`` (no supervisor needed).
99+
# See apache/airflow#51816 and apache/airflow#59093.
100+
return await self.get_xcom_val_af2(key)
89101

90102
async def get_xcom_val_af2(self, key: str) -> Any | None:
91103
from airflow.models import TaskInstance

‎tests/operators/_watcher/test_triggerer.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,24 @@ async def test_get_xcom_val_af3(self):
5252
)
5353
assert result == expected_value
5454

55+
@pytest.mark.skipif(AIRFLOW_VERSION < Version("3.0.0"), reason="Require Airflow >= 3.0.0")
56+
@pytest.mark.asyncio
57+
async def test_get_xcom_val_af3_falls_back_to_af2_on_import_error(self):
58+
# On Airflow 3.0, XCom.get_one imports SUPERVISOR_COMMS (3.1+ only) and raises
59+
# ImportError outside a supervisor (e.g. dag.test()). The SDK path must fall back to
60+
# the direct-DB ORM path instead of propagating the error. See apache/airflow#51816.
61+
expected_value = {"foo": "bar"}
62+
63+
with patch("cosmos.operators._watcher.triggerer.sync_to_async") as mock_sync_to_async:
64+
mock_get_one = AsyncMock(side_effect=ImportError("cannot import name 'SUPERVISOR_COMMS'"))
65+
mock_sync_to_async.return_value = mock_get_one
66+
67+
with patch.object(self.trigger, "get_xcom_val_af2", AsyncMock(return_value=expected_value)) as mock_af2:
68+
result = await self.trigger.get_xcom_val_af3("test_key")
69+
70+
mock_af2.assert_awaited_once_with("test_key")
71+
assert result == expected_value
72+
5573
@pytest.mark.skipif(AIRFLOW_VERSION >= Version("3.0.0"), reason="Require Airflow < 3.0.0")
5674
@pytest.mark.asyncio
5775
async def test_get_xcom_val_af2(self):

‎tests/test_example_dags.py‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -70,17 +70,6 @@ def get_dag_bag() -> DagBag: # noqa: C901
7070
if AIRFLOW_VERSION >= Version("3.0.0"):
7171
file.writelines("example_cosmos_cleanup_dag.py\n")
7272

73-
if Version("3.0.0") <= AIRFLOW_VERSION < Version("3.1.0"):
74-
# `dag.test()` on Airflow 3.0 runs the WatcherTrigger inline. The trigger calls
75-
# `airflow.sdk.execution_time.xcom.XCom.get_one`, which imports `SUPERVISOR_COMMS`
76-
# from `airflow.sdk.execution_time.task_runner`. That symbol does not exist until
77-
# Airflow 3.1, so the import fails, `dag.test()` re-queues the deferred task
78-
# forever, and the integration job hangs until GH Actions kills it. The watcher
79-
# path works on AF 3.0 in a real triggerer process; this skip only excludes the
80-
# `dag.test()` exerciser. Drop once we either depend on AF >= 3.1 or rework the
81-
# trigger's XCom fetch to not need `SUPERVISOR_COMMS`.
82-
file.writelines("watcher_with_freshness_check.py\n")
83-
8473
if AIRFLOW_VERSION == Version("2.9.0"):
8574
# aiobotocore can't be installed with all the other Cosmos test dependencies in Airflow 2.9
8675
file.writelines("cosmos_example_manifest_dag.py\n")

0 commit comments

Comments
 (0)