Skip to content

Commit 705d0c5

Browse files
committed
Fix XCom.get_value() failing in extra link plugins when SUPERVISOR_COMMS unavailable
closes: apache#59093
1 parent 24a334f commit 705d0c5

1 file changed

Lines changed: 202 additions & 59 deletions

File tree

  • task-sdk/src/airflow/sdk/bases

‎task-sdk/src/airflow/sdk/bases/xcom.py‎

Lines changed: 202 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,74 @@ class BaseXCom:
4949

5050
XCOM_RETURN_KEY = "return_value"
5151

52+
@staticmethod
53+
def _get_supervisor_comms():
54+
"""Get SUPERVISOR_COMMS if available, None otherwise."""
55+
try:
56+
from airflow.sdk.execution_time import task_runner
57+
58+
return getattr(task_runner, "SUPERVISOR_COMMS", None)
59+
except (ImportError, AttributeError):
60+
return None
61+
62+
@classmethod
63+
def _get_xcom_from_db(
64+
cls,
65+
*,
66+
key: str,
67+
dag_id: str,
68+
task_id: str,
69+
run_id: str,
70+
map_index: int | None = None,
71+
include_prior_dates: bool = False,
72+
) -> Any | None:
73+
"""
74+
Retrieve an XCom value directly from the database.
75+
76+
:param key: Key for the XCom.
77+
:param dag_id: DAG ID.
78+
:param task_id: Task ID.
79+
:param run_id: DAG run ID.
80+
:param map_index: Map index for mapped tasks.
81+
:param include_prior_dates: If True, search across prior dates.
82+
:return: Deserialized XCom value or None if not found.
83+
"""
84+
from airflow.models.xcom import XComModel
85+
from airflow.utils.session import create_session
86+
87+
with create_session() as session:
88+
query = XComModel.get_many(
89+
key=key,
90+
run_id=run_id,
91+
dag_ids=dag_id,
92+
task_ids=task_id,
93+
map_indexes=map_index if map_index is not None else None,
94+
include_prior_dates=include_prior_dates,
95+
)
96+
97+
result = session.execute(
98+
query.with_only_columns(
99+
XComModel.run_id,
100+
XComModel.task_id,
101+
XComModel.dag_id,
102+
XComModel.map_index,
103+
XComModel.value,
104+
)
105+
).first()
106+
107+
if result is None:
108+
log.debug(
109+
"No XCom value found in database; returning None.",
110+
key=key,
111+
dag_id=dag_id,
112+
task_id=task_id,
113+
run_id=run_id,
114+
map_index=map_index,
115+
)
116+
return None
117+
118+
return XComModel.deserialize_value(result)
119+
52120
@classmethod
53121
def set(
54122
cls,
@@ -177,6 +245,8 @@ def _get_xcom_db_ref(
177245
If there are no results, *None* is returned. If multiple XCom entries
178246
match the criteria, an arbitrary one is returned.
179247
248+
Falls back to database when SUPERVISOR_COMMS is unavailable.
249+
180250
.. seealso:: ``get_value()`` is a convenience function if you already
181251
have a structured TaskInstance or TaskInstanceKey object available.
182252
@@ -190,22 +260,52 @@ def _get_xcom_db_ref(
190260
:param key: A key for the XCom. If provided, only XCom with matching
191261
keys will be returned. Pass *None* (default) to remove the filter.
192262
"""
193-
from airflow.sdk.execution_time.task_runner import SUPERVISOR_COMMS
194-
195-
msg = SUPERVISOR_COMMS.send(
196-
GetXCom(
197-
key=key,
198-
dag_id=dag_id,
199-
task_id=task_id,
200-
run_id=run_id,
201-
map_index=map_index,
202-
),
203-
)
204-
205-
if not isinstance(msg, XComResult):
206-
raise TypeError(f"Expected XComResult, received: {type(msg)} {msg}")
207-
208-
return msg
263+
comms = cls._get_supervisor_comms()
264+
265+
if comms is not None:
266+
msg = comms.send(
267+
GetXCom(
268+
key=key,
269+
dag_id=dag_id,
270+
task_id=task_id,
271+
run_id=run_id,
272+
map_index=map_index,
273+
),
274+
)
275+
276+
if not isinstance(msg, XComResult):
277+
raise TypeError(f"Expected XComResult, received: {type(msg)} {msg}")
278+
279+
return msg
280+
else:
281+
from airflow.models.xcom import XComModel
282+
from airflow.sdk.execution_time.comms import XComResult
283+
from airflow.utils.session import create_session
284+
285+
with create_session() as session:
286+
query = XComModel.get_many(
287+
key=key,
288+
run_id=run_id,
289+
dag_ids=dag_id,
290+
task_ids=task_id,
291+
map_indexes=map_index if map_index is not None else None,
292+
include_prior_dates=False,
293+
)
294+
295+
xcom_row = session.execute(
296+
query.with_only_columns(
297+
XComModel.run_id,
298+
XComModel.task_id,
299+
XComModel.dag_id,
300+
XComModel.map_index,
301+
XComModel.value,
302+
)
303+
).first()
304+
305+
if xcom_row is None:
306+
return XComResult(key=key, value=None)
307+
308+
return XComResult(key=key, value=xcom_row.value)
209309

210310
@classmethod
211311
def get_one(
@@ -227,6 +327,8 @@ def get_one(
227327
If there are no results, *None* is returned. If multiple XCom entries
228328
match the criteria, an arbitrary one is returned.
229329
330+
Falls back to database when SUPERVISOR_COMMS is unavailable.
331+
230332
.. seealso:: ``get_value()`` is a convenience function if you already
231333
have a structured TaskInstance or TaskInstanceKey object available.
232334
@@ -243,33 +345,43 @@ def get_one(
243345
specified Dag run is returned. If *True*, the latest matching XCom is
244346
returned regardless of the run it belongs to.
245347
"""
246-
from airflow.sdk.execution_time.task_runner import SUPERVISOR_COMMS
247-
248-
msg = SUPERVISOR_COMMS.send(
249-
GetXCom(
348+
comms = cls._get_supervisor_comms()
349+
350+
if comms is not None:
351+
msg = comms.send(
352+
GetXCom(
353+
key=key,
354+
dag_id=dag_id,
355+
task_id=task_id,
356+
run_id=run_id,
357+
map_index=map_index,
358+
include_prior_dates=include_prior_dates,
359+
),
360+
)
361+
362+
if not isinstance(msg, XComResult):
363+
raise TypeError(f"Expected XComResult, received: {type(msg)} {msg}")
364+
365+
if msg.value is not None:
366+
return cls.deserialize_value(msg)
367+
log.warning(
368+
"No XCom value found; defaulting to None.",
369+
key=key,
370+
dag_id=dag_id,
371+
task_id=task_id,
372+
run_id=run_id,
373+
map_index=map_index,
374+
)
375+
return None
376+
else:
377+
return cls._get_xcom_from_db(
250378
key=key,
251379
dag_id=dag_id,
252380
task_id=task_id,
253381
run_id=run_id,
254382
map_index=map_index,
255383
include_prior_dates=include_prior_dates,
256-
),
257-
)
258-
259-
if not isinstance(msg, XComResult):
260-
raise TypeError(f"Expected XComResult, received: {type(msg)} {msg}")
261-
262-
if msg.value is not None:
263-
return cls.deserialize_value(msg)
264-
log.warning(
265-
"No XCom value found; defaulting to None.",
266-
key=key,
267-
dag_id=dag_id,
268-
task_id=task_id,
269-
run_id=run_id,
270-
map_index=map_index,
271-
)
272-
return None
384+
)
273385

274386
@classmethod
275387
def get_all(
@@ -290,6 +402,8 @@ def get_all(
290402
This is particularly useful for getting all XCom values from all map
291403
indexes of a mapped task at once.
292404
405+
Falls back to database when SUPERVISOR_COMMS is unavailable.
406+
293407
:param key: A key for the XCom. Only XComs with this key will be returned.
294408
:param run_id: Dag run ID for the task.
295409
:param dag_id: Dag ID to pull XComs from.
@@ -299,28 +413,57 @@ def get_all(
299413
returned regardless of the run they belong to.
300414
:return: List of all XCom values if found.
301415
"""
302-
from airflow.sdk.execution_time.task_runner import SUPERVISOR_COMMS
303-
304-
msg = SUPERVISOR_COMMS.send(
305-
msg=GetXComSequenceSlice(
306-
key=key,
307-
dag_id=dag_id,
308-
task_id=task_id,
309-
run_id=run_id,
310-
start=None,
311-
stop=None,
312-
step=None,
313-
include_prior_dates=include_prior_dates,
314-
),
315-
)
316-
317-
if not isinstance(msg, XComSequenceSliceResult):
318-
raise TypeError(f"Expected XComSequenceSliceResult, received: {type(msg)} {msg}")
319-
320-
if not msg.root:
321-
return None
322-
323-
return [cls.deserialize_value(_XComValueWrapper(value)) for value in msg.root]
416+
comms = cls._get_supervisor_comms()
417+
418+
if comms is not None:
419+
msg = comms.send(
420+
msg=GetXComSequenceSlice(
421+
key=key,
422+
dag_id=dag_id,
423+
task_id=task_id,
424+
run_id=run_id,
425+
start=None,
426+
stop=None,
427+
step=None,
428+
include_prior_dates=include_prior_dates,
429+
),
430+
)
431+
432+
if not isinstance(msg, XComSequenceSliceResult):
433+
raise TypeError(f"Expected XComSequenceSliceResult, received: {type(msg)} {msg}")
434+
435+
if not msg.root:
436+
return None
437+
438+
return [cls.deserialize_value(_XComValueWrapper(value)) for value in msg.root]
439+
else:
440+
from airflow.models.xcom import XComModel
441+
from airflow.utils.session import create_session
442+
443+
with create_session() as session:
444+
query = XComModel.get_many(
445+
key=key,
446+
run_id=run_id,
447+
dag_ids=dag_id,
448+
task_ids=task_id,
449+
map_indexes=None,
450+
include_prior_dates=include_prior_dates,
451+
)
452+
453+
results = session.execute(
454+
query.with_only_columns(
455+
XComModel.run_id,
456+
XComModel.task_id,
457+
XComModel.dag_id,
458+
XComModel.map_index,
459+
XComModel.value,
460+
)
461+
).all()
462+
463+
if not results:
464+
return None
465+
466+
return [XComModel.deserialize_value(row) for row in results]
324467

325468
@staticmethod
326469
def serialize_value(

0 commit comments

Comments
 (0)