Skip to content

Commit 1193158

Browse files
committed
fix: skip scheduling instead of bulk-failing when serialized DAG is transiently missing
When the scheduler cannot find a DAG in the serialized_dag table while checking task concurrency limits, it previously set all SCHEDULED task instances for that DAG to FAILED via a bulk UPDATE. This caused intermittent mass task failures on multi-scheduler setups (e.g. MWAA) where the serialized DAG may be transiently absent during a DAG file parse cycle. Instead, treat the missing serialized DAG as a transient condition: log a warning, add the dag_id to starved_dags so subsequent tasks for the same DAG are also skipped, and let the scheduler retry on the next iteration. The existing test 'test_queued_task_instances_fails_with_missing_dag' has been updated to reflect the new expected behavior (tasks remain SCHEDULED). A new regression test 'test_missing_serialized_dag_does_not_bulk_fail_tasks' explicitly verifies no bulk-failure occurs. Closes apache#62050
1 parent dba4727 commit 1193158

2 files changed

Lines changed: 59 additions & 12 deletions

File tree

‎airflow-core/src/airflow/jobs/scheduler_job_runner.py‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -752,19 +752,17 @@ def _executable_task_instances_to_queued(self, max_tis: int, session: Session) -
752752
serialized_dag = self.scheduler_dag_bag.get_dag_for_run(
753753
dag_run=task_instance.dag_run, session=session
754754
)
755-
# If the dag is missing, fail the task and continue to the next task.
755+
# If the dag is transiently missing, skip scheduling it this iteration
756+
# and try again next time instead of bulk-failing all scheduled tasks.
757+
# See: https://github.com/apache/airflow/issues/62050
756758
if not serialized_dag:
757-
self.log.error(
758-
"DAG '%s' for task instance %s not found in serialized_dag table",
759+
self.log.warning(
760+
"DAG '%s' for task instance %s not found in serialized_dag table, "
761+
"skipping scheduling for this iteration and will retry next time",
759762
dag_id,
760763
task_instance,
761764
)
762-
session.execute(
763-
update(TI)
764-
.where(TI.dag_id == dag_id, TI.state == TaskInstanceState.SCHEDULED)
765-
.values(state=TaskInstanceState.FAILED)
766-
.execution_options(synchronize_session="fetch")
767-
)
765+
starved_dags.add(dag_id)
768766
continue
769767

770768
task_concurrency_limit: int | None = None

‎airflow-core/tests/unit/jobs/test_scheduler_job.py‎

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1800,8 +1800,14 @@ def test_find_executable_task_instances_in_default_pool(self, dag_maker, mock_ex
18001800
session.rollback()
18011801
session.close()
18021802

1803-
def test_queued_task_instances_fails_with_missing_dag(self, dag_maker, session):
1804-
"""Check that task instances of missing DAGs are failed"""
1803+
def test_queued_task_instances_skips_with_missing_dag(self, dag_maker, session):
1804+
"""Check that task instances of transiently missing DAGs are skipped, not bulk-failed.
1805+
1806+
When the serialized DAG is transiently missing (e.g. during a DAG file parse cycle),
1807+
the scheduler should skip scheduling for that DAG in the current iteration and retry
1808+
next time, rather than bulk-failing all SCHEDULED task instances.
1809+
See: https://github.com/apache/airflow/issues/62050
1810+
"""
18051811
dag_id = "SchedulerJobTest.test_find_executable_task_instances_not_in_dagbag"
18061812
task_id_1 = "dummy"
18071813
task_id_2 = "dummydummy"
@@ -1825,10 +1831,53 @@ def test_queued_task_instances_fails_with_missing_dag(self, dag_maker, session):
18251831
session.flush()
18261832
res = self.job_runner._executable_task_instances_to_queued(max_tis=32, session=session)
18271833
session.flush()
1834+
# No tasks should be queued
18281835
assert len(res) == 0
1836+
# Tasks should remain SCHEDULED (not be bulk-failed)
18291837
tis = dr.get_task_instances(session=session)
18301838
assert len(tis) == 2
1831-
assert all(ti.state == State.FAILED for ti in tis)
1839+
assert all(ti.state == State.SCHEDULED for ti in tis)
1840+
1841+
def test_missing_serialized_dag_does_not_bulk_fail_tasks(self, dag_maker, session):
1842+
"""Regression test for https://github.com/apache/airflow/issues/62050
1843+
1844+
When the serialized DAG is transiently missing, all SCHEDULED task instances for the DAG
1845+
should remain SCHEDULED so the scheduler can pick them up in the next iteration.
1846+
Previously, the scheduler would bulk-fail all SCHEDULED tasks when it couldn't find the
1847+
serialized DAG in the DagBag.
1848+
"""
1849+
dag_id = "SchedulerJobTest.test_missing_serialized_dag_bulk_fails"
1850+
1851+
with dag_maker(dag_id=dag_id, session=session, default_args={"max_active_tis_per_dag": 2}):
1852+
EmptyOperator(task_id="task_a")
1853+
EmptyOperator(task_id="task_b")
1854+
1855+
scheduler_job = Job()
1856+
self.job_runner = SchedulerJobRunner(job=scheduler_job)
1857+
1858+
# Simulate serialized DAG being transiently missing
1859+
self.job_runner.scheduler_dag_bag = mock.MagicMock()
1860+
self.job_runner.scheduler_dag_bag.get_dag_for_run.return_value = None
1861+
1862+
dr = dag_maker.create_dagrun(state=DagRunState.RUNNING)
1863+
for ti in dr.task_instances:
1864+
ti.state = State.SCHEDULED
1865+
session.merge(ti)
1866+
session.flush()
1867+
1868+
res = self.job_runner._executable_task_instances_to_queued(max_tis=32, session=session)
1869+
session.flush()
1870+
1871+
# No tasks should be queued since serialized DAG is missing
1872+
assert len(res) == 0
1873+
# All tasks should remain SCHEDULED — not FAILED
1874+
tis = dr.get_task_instances(session=session)
1875+
assert len(tis) == 2
1876+
for ti in tis:
1877+
assert ti.state == State.SCHEDULED, (
1878+
f"Task {ti.task_id} was {ti.state} but expected SCHEDULED. "
1879+
"The scheduler should not bulk-fail tasks when serialized DAG is transiently missing."
1880+
)
18321881

18331882
def test_nonexistent_pool(self, dag_maker):
18341883
dag_id = "SchedulerJobTest.test_nonexistent_pool"

0 commit comments

Comments
 (0)