Skip to content

Commit e751133

Browse files
committed
Fail queued DagRuns with an unresolvable pinned dag version Closes: apache#70056
1 parent 7a41261 commit e751133

2 files changed

Lines changed: 59 additions & 0 deletions

File tree

‎airflow-core/src/airflow/jobs/scheduler_job_runner.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2788,6 +2788,22 @@ def _update_state(dag: SerializedDAG, dag_run: DagRun):
27882788
backfill_id = dag_run.backfill_id
27892789
dag = dag_run.dag = cached_get_dag(dag_run)
27902790
if not dag:
2791+
if dag_run.bundle_version is not None and dag_run.created_dag_version_id is None:
2792+
# The run was pinned to a specific dag version, and that version row has
2793+
# since been deleted (the FK is ON DELETE SET NULL). This can never resolve
2794+
# on a later loop, unlike a run whose version simply hasn't parsed yet, so
2795+
# retrying forever isn't useful here: fail the run explicitly.
2796+
self.log.error(
2797+
"DagRun %s for DAG '%s' is pinned to bundle_version=%s, but its dag "
2798+
"version has been deleted; marking the run as failed instead of "
2799+
"retrying indefinitely",
2800+
dag_run.run_id,
2801+
dag_run.dag_id,
2802+
dag_run.bundle_version,
2803+
)
2804+
dag_run.set_state(DagRunState.FAILED)
2805+
dag_run.notify_dagrun_state_changed(msg="unresolvable_pinned_version")
2806+
continue
27912807
self.log.error("DAG '%s' not found in serialized_dag table", dag_run.dag_id)
27922808
continue
27932809
active_runs = active_runs_of_dags[(dag_id, backfill_id)]

‎airflow-core/tests/unit/jobs/test_scheduler_job.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9911,6 +9911,49 @@ def test_do_scheduling_multi_team_schedules_task_instances(self, dag_maker, sess
99119911
session.refresh(ti)
99129912
assert ti.state != State.NONE
99139913

9914+
def test_start_queued_dagrun_with_unresolvable_pinned_version_is_marked_failed(self, dag_maker, session):
9915+
with dag_maker(dag_id="test_pinned_version_unresolvable", max_active_runs=1):
9916+
EmptyOperator(task_id="dummy_task")
9917+
dag_run = dag_maker.create_dagrun(
9918+
run_id="queued_with_deleted_pinned_version",
9919+
state=DagRunState.QUEUED,
9920+
run_type=DagRunType.MANUAL,
9921+
session=session,
9922+
)
9923+
dag_run.bundle_version = "0123456789abcdef"
9924+
dag_run.created_dag_version_id = None
9925+
session.merge(dag_run)
9926+
session.flush()
9927+
self.job_runner = SchedulerJobRunner(job=Job(), executors=[self.null_exec])
9928+
self.job_runner._start_queued_dagruns(session)
9929+
session.flush()
9930+
dag_run = session.get(DagRun, dag_run.id)
9931+
assert dag_run.state == DagRunState.FAILED
9932+
assert dag_run.end_date is not None
9933+
9934+
def test_start_queued_dagrun_without_pinned_version_keeps_retrying(self, dag_maker, session, monkeypatch):
9935+
with dag_maker(dag_id="test_unpinned_no_version_yet", max_active_runs=1):
9936+
EmptyOperator(task_id="dummy_task")
9937+
dag_run = dag_maker.create_dagrun(
9938+
run_id="queued_unpinned_no_version",
9939+
state=DagRunState.QUEUED,
9940+
run_type=DagRunType.MANUAL,
9941+
session=session,
9942+
)
9943+
dag_run.bundle_version = None
9944+
dag_run.created_dag_version_id = None
9945+
session.merge(dag_run)
9946+
session.flush()
9947+
monkeypatch.setattr(
9948+
"airflow.models.dagbag.DagVersion.get_latest_version",
9949+
lambda *args, **kwargs: None,
9950+
)
9951+
self.job_runner = SchedulerJobRunner(job=Job(), executors=[self.null_exec])
9952+
self.job_runner._start_queued_dagruns(session)
9953+
session.flush()
9954+
dag_run = session.get(DagRun, dag_run.id)
9955+
assert dag_run.state == DagRunState.QUEUED
9956+
99149957

99159958
@pytest.mark.need_serialized_dag
99169959
def test_schedule_dag_run_with_upstream_skip(dag_maker, session):

0 commit comments

Comments
 (0)