|
18 | 18 | from __future__ import annotations |
19 | 19 |
|
20 | 20 | import datetime |
| 21 | +import re |
21 | 22 | from collections import defaultdict |
22 | 23 | from collections.abc import Mapping |
23 | 24 | from contextlib import contextmanager |
|
88 | 89 | from airflow.utils.types import DagRunTriggeredByType, DagRunType |
89 | 90 |
|
90 | 91 | from tests_common.test_utils import db |
| 92 | +from tests_common.test_utils.asserts import capture_orm_selects |
91 | 93 | from tests_common.test_utils.config import conf_vars |
92 | 94 | from tests_common.test_utils.dag import sync_dag_to_db |
93 | 95 | from tests_common.test_utils.db import clear_db_dags, clear_db_runs |
|
103 | 105 |
|
104 | 106 | pytestmark = [pytest.mark.db_test, pytest.mark.need_serialized_dag] |
105 | 107 |
|
| 108 | +# A joined eager load of TaskInstance.dag_run aliases the table, hence the optional suffix. |
| 109 | +SELECTS_RUN_CONF = re.compile(r"\bdag_run(?:_\d+)?\.conf\b") |
| 110 | + |
106 | 111 | TI = TaskInstance |
107 | 112 | DEFAULT_DATE = pendulum.instance(_DEFAULT_DATE) |
108 | 113 |
|
@@ -795,6 +800,26 @@ def test_get_task_instance_on_empty_dagrun(self, dag_maker, session): |
795 | 800 | ti = dag_run.get_task_instance("test_short_circuit_false") |
796 | 801 | assert ti is None |
797 | 802 |
|
| 803 | + def test_get_task_instances_does_not_fetch_run_conf(self, dag_maker, session): |
| 804 | + """The run conf must not be repeated on every task-instance row of the eager join.""" |
| 805 | + conf = {"payload": "x" * 512} |
| 806 | + with dag_maker(dag_id="test_get_task_instances_run_conf", session=session): |
| 807 | + EmptyOperator(task_id="t1") |
| 808 | + EmptyOperator(task_id="t2") |
| 809 | + dag_run = dag_maker.create_dagrun(conf=conf) |
| 810 | + |
| 811 | + with capture_orm_selects("task_instance") as statements: |
| 812 | + tis = DagRun.fetch_task_instances(dag_id=dag_run.dag_id, run_id=dag_run.run_id, session=session) |
| 813 | + |
| 814 | + assert len(tis) == 2 |
| 815 | + assert statements |
| 816 | + assert all(re.search(r"\bdag_run(?:_\d+)?\.run_type\b", stmt) for stmt in statements), statements |
| 817 | + assert not any(SELECTS_RUN_CONF.search(stmt) for stmt in statements), statements |
| 818 | + |
| 819 | + session.expunge_all() |
| 820 | + ti = DagRun.fetch_task_instances(dag_id=dag_run.dag_id, run_id=dag_run.run_id, session=session)[0] |
| 821 | + assert ti.dag_run.conf == conf |
| 822 | + |
798 | 823 | def test_get_latest_runs(self, dag_maker, session): |
799 | 824 | with dag_maker( |
800 | 825 | dag_id="test_latest_runs_1", schedule=datetime.timedelta(days=1), start_date=DEFAULT_DATE |
@@ -2107,6 +2132,81 @@ def task_2(arg2): ... |
2107 | 2132 | ] |
2108 | 2133 |
|
2109 | 2134 |
|
| 2135 | +def test_scheduling_decisions_never_fetch_run_conf(dag_maker, session): |
| 2136 | + """No task-instance query in the scheduling loop may repeat the run conf on every row.""" |
| 2137 | + |
| 2138 | + @task |
| 2139 | + def upstream(): |
| 2140 | + return [1, 2, 3, 4] |
| 2141 | + |
| 2142 | + @task |
| 2143 | + def downstream(arg): ... |
| 2144 | + |
| 2145 | + with dag_maker(dag_id="test_scheduling_decisions_run_conf", session=session): |
| 2146 | + downstream.expand(arg=upstream()) |
| 2147 | + |
| 2148 | + dr = dag_maker.create_dagrun(conf={"payload": "x" * 512}) |
| 2149 | + ti_upstream = dr.get_task_instance("upstream", session=session) |
| 2150 | + |
| 2151 | + statements: list[str] = [] |
| 2152 | + # While the upstream runs the mapped dependency is evaluated per task instance and the |
| 2153 | + # schedulable states are re-read; once it succeeds the mapped task expands. |
| 2154 | + for upstream_state in (State.RUNNING, State.SUCCESS): |
| 2155 | + ti_upstream.state = upstream_state |
| 2156 | + session.flush() |
| 2157 | + with capture_orm_selects("task_instance") as captured: |
| 2158 | + dr.task_instance_scheduling_decisions(session=session) |
| 2159 | + statements.extend(captured) |
| 2160 | + |
| 2161 | + assert statements |
| 2162 | + assert not any(SELECTS_RUN_CONF.search(stmt) for stmt in statements), [ |
| 2163 | + stmt for stmt in statements if SELECTS_RUN_CONF.search(stmt) |
| 2164 | + ] |
| 2165 | + |
| 2166 | + |
| 2167 | +def test_scheduling_decisions_do_not_lazy_load_dag_run(dag_maker, session): |
| 2168 | + """Skipping the dag_run join must not cost one DagRun query per task instance.""" |
| 2169 | + |
| 2170 | + def _schedule_and_count(dag_id: str, fan_out: int) -> tuple[int, int]: |
| 2171 | + @task |
| 2172 | + def upstream(): |
| 2173 | + return list(range(fan_out)) |
| 2174 | + |
| 2175 | + @task |
| 2176 | + def downstream(arg): ... |
| 2177 | + |
| 2178 | + with dag_maker(dag_id=dag_id, max_active_tasks=64, session=session): |
| 2179 | + downstream.expand(arg=upstream()) |
| 2180 | + |
| 2181 | + dr = dag_maker.create_dagrun(conf={"payload": "x" * 512}) |
| 2182 | + dr.get_task_instance("upstream", session=session).state = State.SUCCESS |
| 2183 | + session.add( |
| 2184 | + TaskMap( |
| 2185 | + dag_id=dr.dag_id, |
| 2186 | + task_id="upstream", |
| 2187 | + run_id=dr.run_id, |
| 2188 | + map_index=-1, |
| 2189 | + length=fan_out, |
| 2190 | + keys=None, |
| 2191 | + ) |
| 2192 | + ) |
| 2193 | + session.flush() |
| 2194 | + |
| 2195 | + with capture_orm_selects("dag_run") as statements: |
| 2196 | + dr.task_instance_scheduling_decisions(session=session) # expands the mapped task |
| 2197 | + dr.task_instance_scheduling_decisions(session=session) # evaluates the expanded instances |
| 2198 | + return len(statements), len(dr.get_task_instances(session=session)) |
| 2199 | + |
| 2200 | + small_queries, small_tis = _schedule_and_count("test_lazy_dag_run_small", 2) |
| 2201 | + large_queries, large_tis = _schedule_and_count("test_lazy_dag_run_large", 8) |
| 2202 | + |
| 2203 | + assert (small_tis, large_tis) == (3, 9) |
| 2204 | + assert small_queries == large_queries, ( |
| 2205 | + f"DagRun queries scaled with the task-instance count ({small_queries} -> {large_queries}); " |
| 2206 | + "a lazy load of TaskInstance.dag_run is firing per task instance" |
| 2207 | + ) |
| 2208 | + |
| 2209 | + |
2110 | 2210 | def test_mapped_literal_length_reduction_adds_removed_state(dag_maker, session): |
2111 | 2211 | """Test that when the length of mapped literal reduces, removed state is added""" |
2112 | 2212 |
|
|
0 commit comments