Skip to content

Commit 687effc

Browse files
committed
Fail queued DagRuns with an unresolvable pinned dag version Closes: apache#70056
1 parent 259fe2d commit 687effc

2 files changed

Lines changed: 59 additions & 0 deletions

File tree

‎airflow-core/src/airflow/jobs/scheduler_job_runner.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2860,6 +2860,22 @@ def _update_state(dag: SerializedDAG, dag_run: DagRun):
28602860
backfill_id = dag_run.backfill_id
28612861
dag = dag_run.dag = cached_get_dag(dag_run)
28622862
if not dag:
2863+
if dag_run.bundle_version is not None and dag_run.created_dag_version_id is None:
2864+
# The run was pinned to a specific dag version, and that version row has
2865+
# since been deleted (the FK is ON DELETE SET NULL). This can never resolve
2866+
# on a later loop, unlike a run whose version simply hasn't parsed yet, so
2867+
# retrying forever isn't useful here: fail the run explicitly.
2868+
self.log.error(
2869+
"DagRun %s for DAG '%s' is pinned to bundle_version=%s, but its dag "
2870+
"version has been deleted; marking the run as failed instead of "
2871+
"retrying indefinitely",
2872+
dag_run.run_id,
2873+
dag_run.dag_id,
2874+
dag_run.bundle_version,
2875+
)
2876+
dag_run.set_state(DagRunState.FAILED)
2877+
dag_run.notify_dagrun_state_changed(msg="unresolvable_pinned_version")
2878+
continue
28632879
self.log.error("DAG '%s' not found in serialized_dag table", dag_run.dag_id)
28642880
continue
28652881
active_runs = active_runs_of_dags[(dag_id, backfill_id)]

‎airflow-core/tests/unit/jobs/test_scheduler_job.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10735,6 +10735,49 @@ def test_do_scheduling_multi_team_schedules_task_instances(self, dag_maker, sess
1073510735
session.refresh(ti)
1073610736
assert ti.state != State.NONE
1073710737

10738+
def test_start_queued_dagrun_with_unresolvable_pinned_version_is_marked_failed(self, dag_maker, session):
10739+
with dag_maker(dag_id="test_pinned_version_unresolvable", max_active_runs=1):
10740+
EmptyOperator(task_id="dummy_task")
10741+
dag_run = dag_maker.create_dagrun(
10742+
run_id="queued_with_deleted_pinned_version",
10743+
state=DagRunState.QUEUED,
10744+
run_type=DagRunType.MANUAL,
10745+
session=session,
10746+
)
10747+
dag_run.bundle_version = "0123456789abcdef"
10748+
dag_run.created_dag_version_id = None
10749+
session.merge(dag_run)
10750+
session.flush()
10751+
self.job_runner = SchedulerJobRunner(job=Job(), executors=[self.null_exec])
10752+
self.job_runner._start_queued_dagruns(session)
10753+
session.flush()
10754+
dag_run = session.get(DagRun, dag_run.id)
10755+
assert dag_run.state == DagRunState.FAILED
10756+
assert dag_run.end_date is not None
10757+
10758+
def test_start_queued_dagrun_without_pinned_version_keeps_retrying(self, dag_maker, session, monkeypatch):
10759+
with dag_maker(dag_id="test_unpinned_no_version_yet", max_active_runs=1):
10760+
EmptyOperator(task_id="dummy_task")
10761+
dag_run = dag_maker.create_dagrun(
10762+
run_id="queued_unpinned_no_version",
10763+
state=DagRunState.QUEUED,
10764+
run_type=DagRunType.MANUAL,
10765+
session=session,
10766+
)
10767+
dag_run.bundle_version = None
10768+
dag_run.created_dag_version_id = None
10769+
session.merge(dag_run)
10770+
session.flush()
10771+
monkeypatch.setattr(
10772+
"airflow.models.dagbag.DagVersion.get_latest_version",
10773+
lambda *args, **kwargs: None,
10774+
)
10775+
self.job_runner = SchedulerJobRunner(job=Job(), executors=[self.null_exec])
10776+
self.job_runner._start_queued_dagruns(session)
10777+
session.flush()
10778+
dag_run = session.get(DagRun, dag_run.id)
10779+
assert dag_run.state == DagRunState.QUEUED
10780+
1073810781

1073910782
@pytest.mark.need_serialized_dag
1074010783
def test_schedule_dag_run_with_upstream_skip(dag_maker, session):

0 commit comments

Comments
 (0)