Skip to content

Commit 4bcc197

Browse files
committed
Stop fetching the run conf once per task instance in the scheduler
A Dag run triggered with a large conf makes scheduler memory scale with the number of task instances in the run. TaskInstance.dag_run is a joined eager load, so the run's conf comes back on every row of every task-instance query the scheduling loop makes, and a mapped task with several hundred instances repeats a multi-megabyte conf on every loop while the run is alive. Closes: apache#71267
1 parent 9636fcb commit 4bcc197

6 files changed

Lines changed: 162 additions & 11 deletions

File tree

‎airflow-core/src/airflow/jobs/scheduler_job_runner.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -823,8 +823,11 @@ def _executable_task_instances_to_queued(self, max_tis: int, session: Session) -
823823
# the run's pinned bundle_version. Lazy loads on transient objects silently return
824824
# None instead of raising DetachedInstanceError. Scope the SELECT to version_data
825825
# (the PK is auto-included) so we read two columns rather than the full row.
826+
# Queueing never reads the run conf, which on a transient TI would read as None
827+
# anyway -- see DagRun.fetch_task_instances.
826828
.options(
827829
joinedload(TI.dag_run)
830+
.defer(DagRun.conf)
828831
.selectinload(DagRun.created_dag_version)
829832
.load_only(DagVersion.version_data)
830833
)

‎airflow-core/src/airflow/models/dagrun.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
Mapped,
6262
declared_attr,
6363
joinedload,
64+
lazyload,
6465
mapped_column,
6566
relationship,
6667
synonym,
@@ -973,7 +974,10 @@ def fetch_task_instances(
973974
"""Return the task instances for this dag run."""
974975
tis = (
975976
select(TI)
976-
.options(joinedload(TI.dag_run))
977+
# TI.dag_run is a joined eager load, so every row of this unbounded query would carry
978+
# its own copy of the run's conf: a large conf on a run with many mapped tasks moves
979+
# hundreds of megabytes per scheduler loop. Nothing here reads it.
980+
.options(joinedload(TI.dag_run).defer(DagRun.conf))
977981
.where(
978982
TI.dag_id == dag_id,
979983
TI.run_id == run_id,
@@ -1732,7 +1736,8 @@ def _expand_mapped_task_if_needed(ti: TI) -> Iterable[TI] | None:
17321736
# Check if any ti changed state
17331737
tis_filter = TI.filter_for_tis(old_states)
17341738
if tis_filter is not None:
1735-
fresh_tis = session.scalars(select(TI).where(tis_filter)).all()
1739+
# Only the states are compared, so skip the dag_run join -- see fetch_task_instances.
1740+
fresh_tis = session.scalars(select(TI).options(lazyload(TI.dag_run)).where(tis_filter)).all()
17361741
changed_tis = any(ti.state != old_states[ti.key] for ti in fresh_tis)
17371742

17381743
return ready_tis, changed_tis, expansion_happened

‎airflow-core/src/airflow/models/taskmap.py‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626

2727
from opentelemetry import trace
2828
from sqlalchemy import CheckConstraint, ForeignKeyConstraint, Integer, String, func, or_, select
29-
from sqlalchemy.orm import Mapped, mapped_column
29+
from sqlalchemy.orm import Mapped, lazyload, mapped_column
3030

3131
from airflow._shared.observability.traces import new_task_run_carrier
3232
from airflow.models.base import COLLATION_ARGS, ID_LEN, TaskInstanceDependencies
@@ -168,7 +168,11 @@ def expand_mapped_task(
168168

169169
state: str | None = None
170170
unmapped_ti: TaskInstance | None = session.scalars(
171-
select(TaskInstance).where(
171+
# One row, but the joined eager load would still bring the run conf along with it;
172+
# dag_run is read below and loads on its own.
173+
select(TaskInstance)
174+
.options(lazyload(TaskInstance.dag_run))
175+
.where(
172176
TaskInstance.dag_id == task.dag_id,
173177
TaskInstance.task_id == task.task_id,
174178
TaskInstance.run_id == run_id,
@@ -280,11 +284,16 @@ def expand_mapped_task(
280284

281285
# Any (old) task instances with inapplicable indexes (>= the total
282286
# number we need) are set to "REMOVED".
283-
query = select(TaskInstance).where(
284-
TaskInstance.dag_id == task.dag_id,
285-
TaskInstance.task_id == task.task_id,
286-
TaskInstance.run_id == run_id,
287-
TaskInstance.map_index >= total_expanded_ti_count,
287+
# Only the state is written here, so skip the dag_run join.
288+
query = (
289+
select(TaskInstance)
290+
.options(lazyload(TaskInstance.dag_run))
291+
.where(
292+
TaskInstance.dag_id == task.dag_id,
293+
TaskInstance.task_id == task.task_id,
294+
TaskInstance.run_id == run_id,
295+
TaskInstance.map_index >= total_expanded_ti_count,
296+
)
288297
)
289298
to_update = session.scalars(with_row_locks(query, of=TaskInstance, session=session, skip_locked=True))
290299
for ti in to_update:

‎airflow-core/src/airflow/ti_deps/deps/mapped_task_upstream_dep.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
from typing import TYPE_CHECKING
2323

2424
from sqlalchemy import select
25+
from sqlalchemy.orm import lazyload
2526

2627
from airflow.ti_deps.deps.base_ti_dep import BaseTIDep
2728
from airflow.utils.state import State, TaskInstanceState
@@ -71,7 +72,11 @@ def _get_dep_statuses(
7172
# to know for the purposes of this check.
7273
mapped_dependency_tis = (
7374
session.scalars(
74-
select(TaskInstance).where(
75+
# Runs once per schedulable task instance and reads only the upstream states,
76+
# so skip the dag_run join -- see DagRun.fetch_task_instances.
77+
select(TaskInstance)
78+
.options(lazyload(TaskInstance.dag_run))
79+
.where(
7580
TaskInstance.task_id.in_(operator.task_id for operator in mapped_dependencies),
7681
TaskInstance.dag_id == ti.dag_id,
7782
TaskInstance.run_id == ti.run_id,

‎airflow-core/tests/unit/jobs/test_scheduler_job.py‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@
150150
from airflow.utils.types import DagRunTriggeredByType, DagRunType
151151

152152
from tests_common.pytest_plugin import AIRFLOW_ROOT_PATH
153-
from tests_common.test_utils.asserts import assert_queries_count, count_queries
153+
from tests_common.test_utils.asserts import assert_queries_count, capture_orm_selects, count_queries
154154
from tests_common.test_utils.config import conf_vars, env_vars
155155
from tests_common.test_utils.dag import create_scheduler_dag, sync_dag_to_db, sync_dags_to_db
156156
from tests_common.test_utils.db import (
@@ -1485,6 +1485,35 @@ def test_find_executable_task_instances_backfill(self, dag_maker):
14851485
assert {x.key for x in queued_tis} == {ti_non_backfill.key, ti_backfill.key}
14861486
session.rollback()
14871487

1488+
def test_executable_task_instances_does_not_fetch_run_conf(self, dag_maker, session):
1489+
"""The run conf must not be repeated on every task-instance row of the critical section."""
1490+
scheduler_job = Job()
1491+
self.job_runner = SchedulerJobRunner(job=scheduler_job)
1492+
1493+
conf = {"payload": "x" * 512}
1494+
with dag_maker(dag_id="test_executable_tis_run_conf", max_active_tasks=16, session=session):
1495+
EmptyOperator(task_id="t1")
1496+
EmptyOperator(task_id="t2")
1497+
dr = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED, conf=conf)
1498+
for ti in dr.task_instances:
1499+
ti.state = State.SCHEDULED
1500+
session.flush()
1501+
1502+
with (
1503+
capture_orm_selects("task_instance") as ti_statements,
1504+
capture_orm_selects("dag_version") as dag_version_statements,
1505+
):
1506+
queued_tis = self.job_runner._executable_task_instances_to_queued(max_tis=32, session=session)
1507+
1508+
assert len(queued_tis) == 2
1509+
assert ti_statements
1510+
selects_run_conf = re.compile(r"\bdag_run(?:_\d+)?\.conf\b")
1511+
assert not any(selects_run_conf.search(stmt) for stmt in ti_statements), ti_statements
1512+
# The run's pinned DagVersion is still eager-loaded in the same call; ExecuteTask.make()
1513+
# reads it off task instances that are transient by then.
1514+
assert any("version_data" in stmt for stmt in dag_version_statements), dag_version_statements
1515+
session.rollback()
1516+
14881517
def test_executable_task_instances_no_per_ti_queries(self, dag_maker, session):
14891518
"""Guard against an N+1 when enqueuing task instances.
14901519

‎airflow-core/tests/unit/models/test_dagrun.py‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from __future__ import annotations
1919

2020
import datetime
21+
import re
2122
from collections import defaultdict
2223
from collections.abc import Mapping
2324
from contextlib import contextmanager
@@ -88,6 +89,7 @@
8889
from airflow.utils.types import DagRunTriggeredByType, DagRunType
8990

9091
from tests_common.test_utils import db
92+
from tests_common.test_utils.asserts import capture_orm_selects
9193
from tests_common.test_utils.config import conf_vars
9294
from tests_common.test_utils.dag import sync_dag_to_db
9395
from tests_common.test_utils.db import clear_db_dags, clear_db_runs
@@ -103,6 +105,9 @@
103105

104106
pytestmark = [pytest.mark.db_test, pytest.mark.need_serialized_dag]
105107

108+
# A joined eager load of TaskInstance.dag_run aliases the table, hence the optional suffix.
109+
SELECTS_RUN_CONF = re.compile(r"\bdag_run(?:_\d+)?\.conf\b")
110+
106111
TI = TaskInstance
107112
DEFAULT_DATE = pendulum.instance(_DEFAULT_DATE)
108113

@@ -795,6 +800,26 @@ def test_get_task_instance_on_empty_dagrun(self, dag_maker, session):
795800
ti = dag_run.get_task_instance("test_short_circuit_false")
796801
assert ti is None
797802

803+
def test_get_task_instances_does_not_fetch_run_conf(self, dag_maker, session):
804+
"""The run conf must not be repeated on every task-instance row of the eager join."""
805+
conf = {"payload": "x" * 512}
806+
with dag_maker(dag_id="test_get_task_instances_run_conf", session=session):
807+
EmptyOperator(task_id="t1")
808+
EmptyOperator(task_id="t2")
809+
dag_run = dag_maker.create_dagrun(conf=conf)
810+
811+
with capture_orm_selects("task_instance") as statements:
812+
tis = DagRun.fetch_task_instances(dag_id=dag_run.dag_id, run_id=dag_run.run_id, session=session)
813+
814+
assert len(tis) == 2
815+
assert statements
816+
assert all(re.search(r"\bdag_run(?:_\d+)?\.run_type\b", stmt) for stmt in statements), statements
817+
assert not any(SELECTS_RUN_CONF.search(stmt) for stmt in statements), statements
818+
819+
session.expunge_all()
820+
ti = DagRun.fetch_task_instances(dag_id=dag_run.dag_id, run_id=dag_run.run_id, session=session)[0]
821+
assert ti.dag_run.conf == conf
822+
798823
def test_get_latest_runs(self, dag_maker, session):
799824
with dag_maker(
800825
dag_id="test_latest_runs_1", schedule=datetime.timedelta(days=1), start_date=DEFAULT_DATE
@@ -2107,6 +2132,81 @@ def task_2(arg2): ...
21072132
]
21082133

21092134

2135+
def test_scheduling_decisions_never_fetch_run_conf(dag_maker, session):
2136+
"""No task-instance query in the scheduling loop may repeat the run conf on every row."""
2137+
2138+
@task
2139+
def upstream():
2140+
return [1, 2, 3, 4]
2141+
2142+
@task
2143+
def downstream(arg): ...
2144+
2145+
with dag_maker(dag_id="test_scheduling_decisions_run_conf", session=session):
2146+
downstream.expand(arg=upstream())
2147+
2148+
dr = dag_maker.create_dagrun(conf={"payload": "x" * 512})
2149+
ti_upstream = dr.get_task_instance("upstream", session=session)
2150+
2151+
statements: list[str] = []
2152+
# While the upstream runs the mapped dependency is evaluated per task instance and the
2153+
# schedulable states are re-read; once it succeeds the mapped task expands.
2154+
for upstream_state in (State.RUNNING, State.SUCCESS):
2155+
ti_upstream.state = upstream_state
2156+
session.flush()
2157+
with capture_orm_selects("task_instance") as captured:
2158+
dr.task_instance_scheduling_decisions(session=session)
2159+
statements.extend(captured)
2160+
2161+
assert statements
2162+
assert not any(SELECTS_RUN_CONF.search(stmt) for stmt in statements), [
2163+
stmt for stmt in statements if SELECTS_RUN_CONF.search(stmt)
2164+
]
2165+
2166+
2167+
def test_scheduling_decisions_do_not_lazy_load_dag_run(dag_maker, session):
2168+
"""Skipping the dag_run join must not cost one DagRun query per task instance."""
2169+
2170+
def _schedule_and_count(dag_id: str, fan_out: int) -> tuple[int, int]:
2171+
@task
2172+
def upstream():
2173+
return list(range(fan_out))
2174+
2175+
@task
2176+
def downstream(arg): ...
2177+
2178+
with dag_maker(dag_id=dag_id, max_active_tasks=64, session=session):
2179+
downstream.expand(arg=upstream())
2180+
2181+
dr = dag_maker.create_dagrun(conf={"payload": "x" * 512})
2182+
dr.get_task_instance("upstream", session=session).state = State.SUCCESS
2183+
session.add(
2184+
TaskMap(
2185+
dag_id=dr.dag_id,
2186+
task_id="upstream",
2187+
run_id=dr.run_id,
2188+
map_index=-1,
2189+
length=fan_out,
2190+
keys=None,
2191+
)
2192+
)
2193+
session.flush()
2194+
2195+
with capture_orm_selects("dag_run") as statements:
2196+
dr.task_instance_scheduling_decisions(session=session) # expands the mapped task
2197+
dr.task_instance_scheduling_decisions(session=session) # evaluates the expanded instances
2198+
return len(statements), len(dr.get_task_instances(session=session))
2199+
2200+
small_queries, small_tis = _schedule_and_count("test_lazy_dag_run_small", 2)
2201+
large_queries, large_tis = _schedule_and_count("test_lazy_dag_run_large", 8)
2202+
2203+
assert (small_tis, large_tis) == (3, 9)
2204+
assert small_queries == large_queries, (
2205+
f"DagRun queries scaled with the task-instance count ({small_queries} -> {large_queries}); "
2206+
"a lazy load of TaskInstance.dag_run is firing per task instance"
2207+
)
2208+
2209+
21102210
def test_mapped_literal_length_reduction_adds_removed_state(dag_maker, session):
21112211
"""Test that when the length of mapped literal reduces, removed state is added"""
21122212

0 commit comments

Comments
 (0)