Skip to content

Commit ce01a73

Browse files
authored
Fix scheduler memory spike when a Dag run has a large conf (apache#73907)
* Add tests for the run conf loaded per task instance when scheduling TaskInstance.dag_run is a joined eager load, so a task-instance query that also selects dag_run.conf returns and decodes one copy of the conf per row. A scheduling pass does that twice: fetch_task_instances reads every task instance of the run, and _get_ready_tis re-reads the ones still waiting on their upstreams. A large conf on a run with many mapped task instances multiplies scheduler memory this way. * Fix scheduler memory spike when a Dag run has a large conf TaskInstance.dag_run is a joined eager load, so every row of a task-instance query also carries the run's conf. The scheduler fetches all task instances of each running Dag run on every loop, and re-reads the ones still waiting on their upstreams, so a large conf was decoded once per task instance: a 1 MB conf on a run with 500 mapped task instances took the scheduler from 150 MB to over 3 GB. Neither query needs the conf up front: the few callers that read it can load it on access, and the re-check only compares states.
1 parent d88ca31 commit ce01a73

2 files changed

Lines changed: 58 additions & 3 deletions

File tree

‎airflow-core/src/airflow/models/dagrun.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@
6363
Mapped,
6464
declared_attr,
6565
joinedload,
66+
lazyload,
6667
mapped_column,
6768
relationship,
6869
synonym,
@@ -1064,8 +1065,10 @@ def fetch_task_instances(
10641065
If ``columns`` is provided then the result is plain rows with the
10651066
values of these columns, otherwise the result is ORM objects.
10661067
"""
1068+
# The joined run comes back on every row. Its conf can be large, so it loads on first
1069+
# access instead of being decoded once per task instance.
10671070
stmt = (
1068-
(select(*columns) if columns else select(TI).options(joinedload(TI.dag_run)))
1071+
(select(*columns) if columns else select(TI).options(joinedload(TI.dag_run).defer(DagRun.conf)))
10691072
.where(TI.dag_id == dag_id, TI.run_id == run_id)
10701073
.order_by(TI.task_id, TI.region_id, TI.region_index)
10711074
)
@@ -1975,7 +1978,10 @@ def _expand_mapped_task_if_needed(ti: TI) -> Iterable[TI] | None:
19751978

19761979
# Check if any ti changed state
19771980
if old_states:
1978-
fresh_tis = session.scalars(select(TI).where(TI.id.in_(old_states))).all()
1981+
# Joining the run here would decode its conf once per row; only the states are needed.
1982+
fresh_tis = session.scalars(
1983+
select(TI).options(lazyload(TI.dag_run)).where(TI.id.in_(old_states))
1984+
).all()
19791985
changed_tis = any(ti.state != old_states[ti.id] for ti in fresh_tis)
19801986

19811987
return ready_tis, changed_tis, expansion_happened

‎airflow-core/tests/unit/models/test_dagrun.py‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from __future__ import annotations
1919

2020
import datetime
21+
import re
2122
from collections import defaultdict
2223
from collections.abc import Mapping
2324
from concurrent.futures import ThreadPoolExecutor
@@ -116,7 +117,7 @@
116117
from airflow.utils.types import DagRunTriggeredByType, DagRunType
117118

118119
from tests_common.test_utils import db
119-
from tests_common.test_utils.asserts import assert_queries_count
120+
from tests_common.test_utils.asserts import assert_queries_count, capture_orm_selects
120121
from tests_common.test_utils.config import conf_vars
121122
from tests_common.test_utils.dag import sync_dag_to_db
122123
from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags, clear_db_xcom
@@ -134,6 +135,8 @@
134135

135136
TI = TaskInstance
136137
DEFAULT_DATE = pendulum.instance(_DEFAULT_DATE)
138+
# A joined eager load of TaskInstance.dag_run aliases the table, hence the optional suffix.
139+
SELECTS_RUN_CONF = re.compile(r"\bdag_run(?:_\d+)?\.conf\b")
137140

138141

139142
def get_callback_ti(ti):
@@ -1132,6 +1135,52 @@ def test_get_task_instance_on_empty_dagrun(self, dag_maker, session):
11321135
ti = dag_run.get_task_instance("test_short_circuit_false")
11331136
assert ti is None
11341137

1138+
def test_fetch_task_instances_does_not_select_the_run_conf(self, dag_maker, session):
1139+
"""
1140+
The run's conf must not come back on every task-instance row.
1141+
1142+
``TaskInstance.dag_run`` is a joined eager load, so a query that selected ``dag_run.conf``
1143+
would return and decode one copy of the conf per task instance.
1144+
"""
1145+
conf = {"payload": "x" * 1024}
1146+
with dag_maker(session=session):
1147+
EmptyOperator(task_id="t1")
1148+
EmptyOperator(task_id="t2")
1149+
dag_run = dag_maker.create_dagrun(conf=conf)
1150+
dag_id, run_id = dag_run.dag_id, dag_run.run_id
1151+
session.expunge_all()
1152+
1153+
with capture_orm_selects("task_instance") as statements:
1154+
tis = DagRun.fetch_task_instances(dag_id=dag_id, run_id=run_id, session=session)
1155+
1156+
assert [ti.task_id for ti in tis] == ["t1", "t2"]
1157+
assert statements
1158+
assert not any(SELECTS_RUN_CONF.search(sql) for sql in statements), statements
1159+
# The run is still joined, and its conf loads when read.
1160+
assert tis[0].dag_run.run_id == run_id
1161+
assert tis[0].dag_run.conf == conf
1162+
1163+
def test_scheduling_decisions_do_not_select_the_run_conf(self, dag_maker, session):
1164+
"""
1165+
A scheduling pass must not load the run's conf once per task instance.
1166+
1167+
``downstream`` waits for ``upstream``, so besides fetching the run's task instances the pass
1168+
re-reads the waiting ones to see whether their state changed.
1169+
"""
1170+
with dag_maker(session=session):
1171+
EmptyOperator(task_id="upstream") >> EmptyOperator(task_id="downstream")
1172+
dag_run = dag_maker.create_dagrun(conf={"payload": "x" * 1024})
1173+
1174+
with capture_orm_selects("task_instance") as statements:
1175+
decision = dag_run.task_instance_scheduling_decisions(session=session)
1176+
1177+
assert [ti.task_id for ti in decision.schedulable_tis] == ["upstream"]
1178+
assert statements
1179+
selecting_conf = [sql for sql in statements if SELECTS_RUN_CONF.search(sql)]
1180+
assert not selecting_conf, (
1181+
f"{len(selecting_conf)} of {len(statements)} task-instance queries select the run conf"
1182+
)
1183+
11351184
def test_get_latest_runs(self, dag_maker, session):
11361185
with dag_maker(
11371186
dag_id="test_latest_runs_1", schedule=datetime.timedelta(days=1), start_date=DEFAULT_DATE

0 commit comments

Comments
 (0)