Skip to content

Commit ce74083

Browse files
committed
Keep scheduled tasks when serialized DAG is missing (apache#62050)
1 parent e84b78e commit ce74083

2 files changed

Lines changed: 24 additions & 8 deletions

File tree

‎airflow-core/src/airflow/jobs/scheduler_job_runner.py‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -587,6 +587,7 @@ def _task_concurrency_allows_execution(
587587
task_instance: TI,
588588
concurrency_map: ConcurrencyMap,
589589
session: Session,
590+
starved_dags: set[str],
590591
starved_tasks: set[tuple[str, str]],
591592
starved_tasks_task_dagrun_concurrency: set[tuple[str, str, str]],
592593
) -> bool:
@@ -600,21 +601,17 @@ def _task_concurrency_allows_execution(
600601
session=session,
601602
)
602603

603-
# If the DAG is missing, fail all scheduled TIs for this DAG.
604+
# A transient gap in serialized DAG availability must not destroy every
605+
# scheduled task for the DAG. Exclude it from this batch so other DAGs
606+
# can still use the available scheduling slots, then retry next tick.
604607
if not serialized_dag:
605608
self.log.error(
606609
"DAG '%s' for task instance %s not found in serialized_dag table",
607610
dag_id,
608611
task_instance,
609612
)
610613

611-
session.execute(
612-
update(TI)
613-
.where(TI.dag_id == dag_id, TI.state == TaskInstanceState.SCHEDULED)
614-
.values(state=TaskInstanceState.FAILED)
615-
.execution_options(synchronize_session="fetch")
616-
)
617-
614+
starved_dags.add(dag_id)
618615
return False
619616

620617
if not serialized_dag.has_task(task_id):
@@ -987,6 +984,7 @@ def _executable_task_instances_to_queued(self, max_tis: int, session: Session) -
987984
task_instance=task_instance,
988985
concurrency_map=concurrency_map,
989986
session=session,
987+
starved_dags=starved_dags,
990988
starved_tasks=starved_tasks,
991989
starved_tasks_task_dagrun_concurrency=(starved_tasks_task_dagrun_concurrency),
992990
)

‎airflow-core/tests/unit/jobs/test_scheduler_job.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1466,6 +1466,24 @@ def test_find_executable_task_instances_backfill(self, dag_maker):
14661466
assert {x.key for x in queued_tis} == {ti_non_backfill.key, ti_backfill.key}
14671467
session.rollback()
14681468

1469+
def test_missing_serialized_dag_keeps_scheduled_tasks_for_a_later_tick(self, dag_maker, session):
1470+
"""A transient serialization gap must not fail every scheduled task for a DAG."""
1471+
with dag_maker(dag_id="missing_serialized_dag", max_active_tasks=16):
1472+
task = EmptyOperator(task_id="task", max_active_tis_per_dag=1)
1473+
1474+
dag_run = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED)
1475+
task_instance = dag_run.get_task_instance(task.task_id, session=session)
1476+
task_instance.state = State.SCHEDULED
1477+
session.flush()
1478+
1479+
self.job_runner = SchedulerJobRunner(job=Job(), executors=[self.null_exec])
1480+
self.job_runner.scheduler_dag_bag = mock.MagicMock()
1481+
self.job_runner.scheduler_dag_bag.get_dag_for_run.return_value = None
1482+
1483+
assert self.job_runner._executable_task_instances_to_queued(max_tis=32, session=session) == []
1484+
session.refresh(task_instance)
1485+
assert task_instance.state == State.SCHEDULED
1486+
14691487
def test_executable_task_instances_no_per_ti_queries(self, dag_maker, session):
14701488
"""Guard against an N+1 when enqueuing task instances.
14711489

0 commit comments

Comments
 (0)