Skip to content

Commit 7920321

Browse files
committed
SQLA: Replace the deprecated lazy="noload" with lazy="raise"
SQLAlchemy 2.1 deprecated the `noload` lazy loading strategy (sqlalchemy/sqlalchemy#11045). `noload` silently returns `None`/empty collections — essentially incorrect results — and will be removed in a future release. This PR replaces all 5 occurrences of `lazy="noload"` with `lazy="raise"`, which raises `InvalidRequestError` if the relationship is accessed without an explicit eager load (e.g. `joinedload`). All affected relationships are already properly loaded via `joinedload()` wherever they're accessed, so this is a safe drop-in that also catches missing eager loads at development time instead of silently returning `None`. Two callers needed fixes to work correctly with `lazy="raise"`: - `TaskInstance.rendered_task_instance_fields` and `TaskInstance.hitl_detail` needed `passive_deletes=True` to tell SQLAlchemy to rely on the DB-level `ON DELETE CASCADE` rather than attempting ORM-level cascade processing (which would fail since FK columns are also PK columns on RTIF, and `lazy="raise"` prevents the ORM from loading the collection to clear them). - The HITL API endpoints needed `joinedload(TI.rendered_task_instance_fields)` added to their queries, since `TaskInstanceResponse` accesses `rendered_task_instance_fields` during Pydantic serialization. The `get_hitl_detail` endpoint also needed an explicit `model_validate()` call so serialization happens while the session is still active. **Changed models:** - `Log.task_instance` - `TaskInstance.rendered_task_instance_fields` - `TaskInstance.hitl_detail` - `TaskInstanceHistory.hitl_detail` - `XComModel.task` related: apache#61229
1 parent de332af commit 7920321

11 files changed

Lines changed: 119 additions & 12 deletions

File tree

‎airflow-core/src/airflow/api_fastapi/common/db/task_instances.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,5 +52,8 @@ def eager_load_TI_and_TIH_for_validation(
5252
contains_eager(orm_model.dag_version).options(joinedload(DagVersion.bundle)),
5353
)
5454
if orm_model is TaskInstance:
55-
query = query.options(joinedload(orm_model.task_instance_note))
55+
query = query.options(
56+
joinedload(orm_model.task_instance_note),
57+
joinedload(orm_model.rendered_task_instance_fields),
58+
)
5659
return query

‎airflow-core/src/airflow/api_fastapi/core_api/routes/public/hitl.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,9 @@ def _get_task_instance_with_hitl_detail(
8484
try_number: int | None = None,
8585
) -> TI | TIH:
8686
def _query(orm_object: Base) -> TI | TIH | None:
87+
options = [joinedload(orm_object.hitl_detail)]
88+
if orm_object is TI:
89+
options.append(joinedload(TI.rendered_task_instance_fields))
8790
query = (
8891
select(orm_object)
8992
.where(
@@ -92,7 +95,7 @@ def _query(orm_object: Base) -> TI | TIH | None:
9295
orm_object.task_id == task_id,
9396
orm_object.map_index == map_index,
9497
)
95-
.options(joinedload(orm_object.hitl_detail))
98+
.options(*options)
9699
)
97100

98101
if try_number is not None:
@@ -213,7 +216,7 @@ def get_hitl_detail(
213216
map_index=map_index,
214217
try_number=None,
215218
)
216-
return task_instance.hitl_detail
219+
return HITLDetail.model_validate(task_instance.hitl_detail)
217220

218221

219222
@task_instances_hitl_router.get(
@@ -304,6 +307,7 @@ def get_hitl_details(
304307
joinedload(TI.dag_run).joinedload(DagRun.dag_model),
305308
joinedload(TI.task_instance_note),
306309
joinedload(TI.dag_version).joinedload(DagVersion.bundle),
310+
joinedload(TI.rendered_task_instance_fields),
307311
),
308312
)
309313
)

‎airflow-core/src/airflow/api_fastapi/core_api/routes/public/task_instances.py‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
from __future__ import annotations
1919

20+
from collections.abc import Sequence
2021
from typing import Annotated, Literal, cast
2122

2223
import structlog
@@ -637,10 +638,6 @@ def get_task_instances_batch(
637638
limit=limit,
638639
session=session,
639640
)
640-
task_instance_select = task_instance_select.options(
641-
joinedload(TI.rendered_task_instance_fields),
642-
)
643-
644641
task_instances = session.scalars(task_instance_select)
645642

646643
return TaskInstanceCollectionResponse(
@@ -802,6 +799,7 @@ def _collect_relatives(run_id: str, direction: Literal["upstream", "downstream"]
802799
*((t, m) for t, m in mapped_tasks_tuples if t not in normal_task_ids),
803800
]
804801

802+
task_instances: Sequence[TI]
805803
if dag_run_id is not None and not (past or future):
806804
# Use run_id-based clearing when we have a specific dag_run_id and not using past/future
807805
task_instances = dag.clear(
@@ -845,6 +843,21 @@ def _collect_relatives(run_id: str, direction: Literal["upstream", "downstream"]
845843
user=user,
846844
)
847845

846+
# Eagerly load rendered_task_instance_fields for serialization (lazy='raise' prevents lazy access).
847+
# dag.clear() returns TIs without this relationship loaded; re-query with joinedload.
848+
# populate_existing=True ensures the joinedload updates TIs already in the identity map.
849+
if task_instances:
850+
task_instances = (
851+
session.scalars(
852+
select(TI)
853+
.options(joinedload(TI.rendered_task_instance_fields))
854+
.where(TI.id.in_([ti.id for ti in task_instances]))
855+
.execution_options(populate_existing=True)
856+
)
857+
.unique()
858+
.all()
859+
)
860+
848861
return TaskInstanceCollectionResponse(
849862
task_instances=[TaskInstanceResponse.model_validate(ti) for ti in task_instances],
850863
total_entries=len(task_instances),
@@ -878,6 +891,7 @@ def patch_task_instance_dry_run(
878891
update_mask: list[str] | None = Query(None),
879892
) -> TaskInstanceCollectionResponse:
880893
"""Update a task instance dry_run mode."""
894+
tis: Sequence[TI]
881895
dag, tis, data = _patch_ti_validate_request(
882896
dag_id, dag_run_id, task_id, dag_bag, body, session, map_index, update_mask
883897
)
@@ -899,6 +913,21 @@ def patch_task_instance_dry_run(
899913
or []
900914
)
901915

916+
# Eagerly load rendered_task_instance_fields for serialization (lazy='raise' prevents lazy access).
917+
# dag.set_task_instance_state() may return TIs without this relationship loaded.
918+
# populate_existing=True ensures the joinedload updates TIs already in the identity map.
919+
if tis:
920+
tis = (
921+
session.scalars(
922+
select(TI)
923+
.options(joinedload(TI.rendered_task_instance_fields))
924+
.where(TI.id.in_([ti.id for ti in tis]))
925+
.execution_options(populate_existing=True)
926+
)
927+
.unique()
928+
.all()
929+
)
930+
902931
return TaskInstanceCollectionResponse(
903932
task_instances=[
904933
TaskInstanceResponse.model_validate(

‎airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
from fastapi import Depends, status
2424
from sqlalchemy import and_, func, select
25+
from sqlalchemy.orm import defaultload
2526

2627
from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity
2728
from airflow.api_fastapi.common.db.common import (
@@ -219,6 +220,9 @@ def get_dags(
219220
HITLDetail,
220221
)
221222
.join(TaskInstance, HITLDetail.ti_id == TaskInstance.id)
223+
.options(
224+
defaultload(HITLDetail.task_instance).joinedload(TaskInstance.rendered_task_instance_fields)
225+
)
222226
.where(
223227
HITLDetail.responded_at.is_(None),
224228
TaskInstance.state == TaskInstanceState.DEFERRED,

‎airflow-core/src/airflow/models/log.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ class Log(Base):
6363
viewonly=True,
6464
foreign_keys=[dag_id, task_id, run_id, map_index],
6565
primaryjoin="and_(Log.dag_id == TaskInstance.dag_id, Log.task_id == TaskInstance.task_id, Log.run_id == TaskInstance.run_id, Log.map_index == TaskInstance.map_index)",
66-
lazy="noload",
66+
lazy="raise",
6767
)
6868

6969
__table_args__ = (

‎airflow-core/src/airflow/models/taskinstance.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -614,8 +614,10 @@ class TaskInstance(Base, LoggingMixin, BaseWorkload):
614614
trigger = relationship("Trigger", uselist=False, back_populates="task_instance")
615615
triggerer_job = association_proxy("trigger", "triggerer_job")
616616
dag_run = relationship("DagRun", back_populates="task_instances", lazy="joined", innerjoin=True)
617-
rendered_task_instance_fields = relationship("RenderedTaskInstanceFields", lazy="noload", uselist=False)
618-
hitl_detail = relationship("HITLDetail", lazy="noload", uselist=False)
617+
rendered_task_instance_fields = relationship(
618+
"RenderedTaskInstanceFields", lazy="raise", uselist=False, passive_deletes=True
619+
)
620+
hitl_detail = relationship("HITLDetail", lazy="raise", uselist=False, passive_deletes=True)
619621

620622
run_after = association_proxy("dag_run", "run_after")
621623
logical_date = association_proxy("dag_run", "logical_date")

‎airflow-core/src/airflow/models/taskinstancehistory.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ class TaskInstanceHistory(Base):
131131
foreign_keys=[run_id, dag_id],
132132
)
133133

134-
hitl_detail = relationship("HITLDetailHistory", lazy="noload", uselist=False)
134+
hitl_detail = relationship("HITLDetailHistory", lazy="raise", uselist=False)
135135

136136
def __init__(
137137
self,

‎airflow-core/src/airflow/models/xcom.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ class XComModel(TaskInstanceDependencies):
106106
task = relationship(
107107
"TaskInstance",
108108
viewonly=True,
109-
lazy="noload",
109+
lazy="raise",
110110
)
111111

112112
@classmethod

‎airflow-core/tests/unit/models/test_log.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import pytest
2121
from sqlalchemy import select
22+
from sqlalchemy.exc import InvalidRequestError
2223
from sqlalchemy.orm import joinedload
2324

2425
from airflow.models.log import Log
@@ -29,6 +30,27 @@
2930

3031

3132
class TestLogTaskInstanceReproduction:
33+
def test_log_task_instance_raises_without_joinedload(self, dag_maker, session):
34+
"""Accessing Log.task_instance without joinedload should raise."""
35+
with dag_maker("dag_raise_test", session=session):
36+
EmptyOperator(task_id="task_1")
37+
38+
dr = dag_maker.create_dagrun()
39+
ti = dr.get_task_instance("task_1")
40+
session.merge(ti)
41+
session.commit()
42+
43+
log = Log(event="test_event", task_instance=ti)
44+
session.add(log)
45+
session.commit()
46+
47+
session.expire_all()
48+
stmt = select(Log).where(Log.id == log.id)
49+
loaded_log = session.scalar(stmt)
50+
51+
with pytest.raises(InvalidRequestError):
52+
loaded_log.task_instance
53+
3254
def test_log_task_instance_join_correctness(self, dag_maker, session):
3355
# Create dag_1 with a task
3456
with dag_maker("dag_1", session=session):

‎airflow-core/tests/unit/models/test_taskinstance.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2750,6 +2750,29 @@ def test_defer_task_with_trigger_timeout(create_task_instance):
27502750
assert abs((ti.trigger_timeout - expected_timeout).total_seconds()) < 5
27512751

27522752

2753+
class TestTaskInstanceRelationships:
2754+
@pytest.mark.parametrize(
2755+
"attr",
2756+
["rendered_task_instance_fields", "hitl_detail"],
2757+
)
2758+
def test_noload_relationships_raise_without_joinedload(self, dag_maker, session, attr):
2759+
"""Accessing lazy='raise' relationships without joinedload should raise."""
2760+
from sqlalchemy.exc import InvalidRequestError
2761+
2762+
with dag_maker("test_dag", session=session):
2763+
EmptyOperator(task_id="task_1")
2764+
2765+
dr = dag_maker.create_dagrun()
2766+
ti = dr.get_task_instance("task_1")
2767+
session.merge(ti)
2768+
session.commit()
2769+
2770+
loaded_ti = session.scalar(select(TaskInstance).where(TaskInstance.id == ti.id))
2771+
2772+
with pytest.raises(InvalidRequestError):
2773+
getattr(loaded_ti, attr)
2774+
2775+
27532776
class TestTaskInstanceRecordTaskMapXComPush:
27542777
"""Test TI.xcom_push() correctly records return values for task-mapping."""
27552778

0 commit comments

Comments
 (0)