Skip to content

Commit 7ac3b14

Browse files
committed
Add server-side XComModel value helper
1 parent 2e50d6e commit 7ac3b14

7 files changed

Lines changed: 147 additions & 2 deletions

File tree

‎airflow-core/src/airflow/models/xcom.py‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@
5454
from sqlalchemy.orm import Session
5555
from sqlalchemy.sql.expression import Select, TextClause
5656

57+
from airflow.models.taskinstancekey import TaskInstanceKey
58+
5759

5860
XCOM_RETURN_KEY = "return_value"
5961

@@ -332,6 +334,57 @@ def get_many(
332334
return query.limit(limit)
333335
return query
334336

337+
@classmethod
338+
@provide_session
339+
def get_one(
340+
cls,
341+
*,
342+
key: str,
343+
dag_id: str,
344+
task_id: str,
345+
run_id: str,
346+
map_index: int | None = None,
347+
include_prior_dates: bool = False,
348+
session: Session = NEW_SESSION,
349+
) -> Any | None:
350+
"""Retrieve and deserialize a single XCom value from the metadata database."""
351+
result = session.execute(
352+
cls.get_many(
353+
key=key,
354+
dag_ids=dag_id,
355+
task_ids=task_id,
356+
run_id=run_id,
357+
map_indexes=map_index,
358+
include_prior_dates=include_prior_dates,
359+
limit=1,
360+
).with_only_columns(cls.value)
361+
).first()
362+
if result is None:
363+
return None
364+
from airflow.sdk.bases.xcom import BaseXCom
365+
from airflow.sdk.execution_time.xcom import resolve_xcom_backend
366+
367+
XCom = resolve_xcom_backend()
368+
if XCom is BaseXCom:
369+
return cls.deserialize_value(result)
370+
return XCom.deserialize_value(result)
371+
372+
@classmethod
373+
def get_value(
374+
cls,
375+
*,
376+
ti_key: TaskInstanceKey,
377+
key: str,
378+
) -> Any | None:
379+
"""Retrieve and deserialize an XCom value for a task instance key."""
380+
return cls.get_one(
381+
key=key,
382+
dag_id=ti_key.dag_id,
383+
task_id=ti_key.task_id,
384+
run_id=ti_key.run_id,
385+
map_index=ti_key.map_index,
386+
)
387+
335388
@staticmethod
336389
def serialize_value(
337390
value: Any,

‎airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_extra_links.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,21 @@ class TryNumberPlugin(AirflowPlugin):
7272
operator_extra_links = [TryNumberLink()]
7373

7474

75+
class XComModelLink(BaseOperatorLink):
76+
name = "XCom Model"
77+
operators = [CustomOperator]
78+
79+
def get_link(self, operator, *, ti_key):
80+
from airflow.providers.common.compat.sdk import XComModel
81+
82+
return XComModel.get_value(key="plugin_link", ti_key=ti_key) or ""
83+
84+
85+
class XComModelPlugin(AirflowPlugin):
86+
name = "xcom_model_plugin"
87+
operator_extra_links = [XComModelLink()]
88+
89+
7590
@pytest.mark.mock_plugin_manager(plugins=[])
7691
class TestGetExtraLinks:
7792
dag_id = "TEST_DAG_ID"
@@ -265,6 +280,23 @@ def test_should_respond_200_support_plugins(self, test_client):
265280
).model_dump()
266281
)
267282

283+
@pytest.mark.mock_plugin_manager(plugins=[XComModelPlugin])
284+
def test_plugin_link_can_read_xcom_with_xcommodel(self, test_client):
285+
XCom.set(
286+
key="plugin_link",
287+
value="https://example.com/plugin-link",
288+
task_id=self.task_single_link,
289+
dag_id=self.dag_id,
290+
run_id=self.dag_run_id,
291+
)
292+
293+
response = test_client.get(
294+
f"/dags/{self.dag_id}/dagRuns/{self.dag_run_id}/taskInstances/{self.task_single_link}/links",
295+
)
296+
297+
assert response.status_code == 200
298+
assert response.json()["extra_links"]["XCom Model"] == "https://example.com/plugin-link"
299+
268300
def test_should_respond_200_mapped_task_instance(self, test_client, session):
269301
for map_index, value in enumerate(["TEST_LINK_VALUE_1", "TEST_LINK_VALUE_2"]):
270302
XCom.set(

‎airflow-core/tests/unit/models/test_xcom.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,16 @@ def test_xcom_get_one(self, session, task_instance):
240240
).first()
241241
assert XComModel.deserialize_value(stored_value) == {"key": "value"}
242242

243+
def test_xcommodel_get_value(self, task_instance, push_simple_json_xcom):
244+
push_simple_json_xcom(ti=task_instance, key="xcom_1", value={"key": "value"})
245+
246+
value = XComModel.get_value(
247+
ti_key=task_instance.key,
248+
key="xcom_1",
249+
)
250+
251+
assert value == {"key": "value"}
252+
243253
@pytest.fixture
244254
def tis_for_xcom_get_one_from_prior_date(self, task_instance_factory, push_simple_json_xcom):
245255
date1 = timezone.datetime(2021, 12, 3, 4, 56)

‎providers/common/compat/src/airflow/providers/common/compat/sdk.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
if TYPE_CHECKING:
3131
import airflow.sdk.io as io # noqa: F401
3232
import airflow.sdk.timezone as timezone # noqa: F401
33-
from airflow.models.xcom import XCOM_RETURN_KEY as XCOM_RETURN_KEY
33+
from airflow.models.xcom import XCOM_RETURN_KEY as XCOM_RETURN_KEY, XComModel as XComModel
3434
from airflow.sdk import (
3535
DAG as DAG,
3636
Asset as Asset,
@@ -135,6 +135,7 @@
135135
"AssetAlias": ("airflow.sdk", "airflow.datasets", "DatasetAlias"),
136136
"AssetAll": ("airflow.sdk", "airflow.datasets", "DatasetAll"),
137137
"AssetAny": ("airflow.sdk", "airflow.datasets", "DatasetAny"),
138+
"XComModel": ("airflow.models.xcom", "airflow.models.xcom", "XCom"),
138139
}
139140

140141
# Airflow 3-only renames (not available in Airflow 2)

‎providers/common/compat/tests/unit/common/compat/test_sdk.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,3 +52,16 @@ def test_invalid_import_raises_attribute_error():
5252

5353
with pytest.raises(AttributeError, match="has no attribute 'NonExistentClass'"):
5454
_ = sdk.NonExistentClass
55+
56+
57+
def test_xcom_model_imports_server_side_model_or_airflow2_fallback():
58+
from airflow.models.xcom import XCom
59+
from airflow.providers.common.compat import sdk
60+
from airflow.providers.common.compat.version_compat import AIRFLOW_V_3_0_PLUS
61+
62+
if AIRFLOW_V_3_0_PLUS:
63+
from airflow.models.xcom import XComModel
64+
65+
assert sdk.XComModel is XComModel
66+
else:
67+
assert sdk.XComModel is XCom

‎providers/common/io/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ requires-python = ">=3.10"
6060
# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
6161
dependencies = [
6262
"apache-airflow>=2.11.0",
63-
"apache-airflow-providers-common-compat>=1.12.0",
63+
"apache-airflow-providers-common-compat>=1.12.0", # use next version
6464
]
6565

6666
# The optional dependencies should be modified in place in the generated file

‎providers/common/io/tests/unit/common/io/xcom/test_backend.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
# under the License.
1818
from __future__ import annotations
1919

20+
import inspect
2021
from unittest.mock import MagicMock, patch
2122

2223
import pytest
@@ -44,6 +45,12 @@
4445
from airflow.models.xcom import BaseXCom, resolve_xcom_backend # type: ignore[no-redef]
4546

4647

48+
def _xcommodel_supports_get_value_backend_regression() -> bool:
49+
if not AIRFLOW_V_3_0_PLUS or not hasattr(XComModel, "get_value"):
50+
return False
51+
return "serialize" in inspect.signature(XComModel.set).parameters
52+
53+
4754
@pytest.fixture(autouse=True)
4855
def reset_db():
4956
"""Reset XCom entries."""
@@ -222,6 +229,35 @@ def test_value_storage(self, task_instance, mock_supervisor_comms, session):
222229
)
223230
assert str(p) == qry.first().value
224231

232+
@pytest.mark.skipif(
233+
not _xcommodel_supports_get_value_backend_regression(),
234+
reason="XComModel.get_value backend regression requires the current server-side XComModel helpers",
235+
)
236+
def test_xcommodel_get_value_uses_configured_backend(self, task_instance, session):
237+
session.add(task_instance)
238+
session.commit()
239+
XCom = resolve_xcom_backend()
240+
airflow.models.xcom.XCom = XCom
241+
242+
expected_value = {"key": "bigvaluebigvaluebigvalue" * 100}
243+
stored_value = XCom.serialize_value(
244+
value=expected_value,
245+
key=XCOM_RETURN_KEY,
246+
dag_id=task_instance.dag_id,
247+
task_id=task_instance.task_id,
248+
run_id=task_instance.run_id,
249+
)
250+
XComModel.set(
251+
key=XCOM_RETURN_KEY,
252+
value=stored_value,
253+
dag_id=task_instance.dag_id,
254+
task_id=task_instance.task_id,
255+
run_id=task_instance.run_id,
256+
serialize=False,
257+
)
258+
259+
assert XComModel.get_value(key=XCOM_RETURN_KEY, ti_key=task_instance.key) == expected_value
260+
225261
def test_clear(self, task_instance, session, mock_supervisor_comms):
226262
session.add(task_instance)
227263
session.commit()

0 commit comments

Comments
 (0)