Skip to content

Commit 47bb09f

Browse files
committed
Fix backfill marked complete before DagRuns are created (apache#61375)
The scheduler's _mark_backfills_complete() could mark a backfill as completed during the window between the Backfill row commit and DagRun creation. Add an EXISTS guard on BackfillDagRun so backfills still being initialized are skipped.
1 parent bfcb77f commit 47bb09f

2 files changed

Lines changed: 56 additions & 2 deletions

File tree

‎airflow-core/src/airflow/jobs/scheduler_job_runner.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@
7171
TaskInletAssetReference,
7272
TaskOutletAssetReference,
7373
)
74-
from airflow.models.backfill import Backfill
74+
from airflow.models.backfill import Backfill, BackfillDagRun
7575
from airflow.models.callback import Callback, CallbackType, ExecutorCallback
7676
from airflow.models.dag import DagModel
7777
from airflow.models.dag_version import DagVersion
@@ -1855,6 +1855,9 @@ def _mark_backfills_complete(self, session: Session = NEW_SESSION) -> None:
18551855
# todo: AIP-78 simplify this function to an update statement
18561856
query = select(Backfill).where(
18571857
Backfill.completed_at.is_(None),
1858+
# Guard: backfill must have at least one BackfillDagRun association,
1859+
# otherwise it is still being set up (see #61375).
1860+
exists(select(BackfillDagRun.id).where(BackfillDagRun.backfill_id == Backfill.id)),
18581861
~exists(
18591862
select(DagRun.id).where(
18601863
and_(DagRun.backfill_id == Backfill.id, DagRun.state.in_(unfinished_states))

‎airflow-core/tests/unit/jobs/test_scheduler_job.py‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@
6565
AssetPartitionDagRun,
6666
PartitionedAssetKeyLog,
6767
)
68-
from airflow.models.backfill import Backfill, _create_backfill
68+
from airflow.models.backfill import Backfill, BackfillDagRun, ReprocessBehavior, _create_backfill
6969
from airflow.models.callback import ExecutorCallback
7070
from airflow.models.dag import DagModel, get_last_dagrun, infer_automated_data_interval
7171
from airflow.models.dag_version import DagVersion
@@ -8877,6 +8877,57 @@ def test_mark_backfills_completed(dag_maker, session):
88778877
assert b.completed_at.timestamp() > 0
88788878

88798879

8880+
def test_mark_backfills_complete_skips_initializing_backfill(dag_maker, session):
8881+
clear_db_backfills()
8882+
dag_id = "test_backfill_race_lifecycle"
8883+
with dag_maker(serialized=True, dag_id=dag_id, schedule="@daily"):
8884+
BashOperator(task_id="hi", bash_command="echo hi")
8885+
b = Backfill(
8886+
dag_id=dag_id,
8887+
from_date=pendulum.parse("2021-01-01"),
8888+
to_date=pendulum.parse("2021-01-03"),
8889+
max_active_runs=10,
8890+
dag_run_conf={},
8891+
reprocess_behavior=ReprocessBehavior.NONE,
8892+
)
8893+
session.add(b)
8894+
session.commit()
8895+
backfill_id = b.id
8896+
session.expunge_all()
8897+
runner = SchedulerJobRunner(
8898+
job=Job(job_type=SchedulerJobRunner.job_type), executors=[MockExecutor(do_update=False)]
8899+
)
8900+
runner._mark_backfills_complete()
8901+
b = session.get(Backfill, backfill_id)
8902+
assert b.completed_at is None
8903+
session.expunge_all()
8904+
dr = DagRun(
8905+
dag_id=dag_id,
8906+
run_id="backfill__2021-01-01T00:00:00+00:00",
8907+
run_type=DagRunType.BACKFILL_JOB,
8908+
logical_date=pendulum.parse("2021-01-01"),
8909+
data_interval=(pendulum.parse("2021-01-01"), pendulum.parse("2021-01-02")),
8910+
run_after=pendulum.parse("2021-01-02"),
8911+
state=DagRunState.SUCCESS,
8912+
backfill_id=backfill_id,
8913+
)
8914+
session.add(dr)
8915+
session.flush()
8916+
session.add(
8917+
BackfillDagRun(
8918+
backfill_id=backfill_id,
8919+
dag_run_id=dr.id,
8920+
logical_date=pendulum.parse("2021-01-01"),
8921+
sort_ordinal=1,
8922+
)
8923+
)
8924+
session.commit()
8925+
session.expunge_all()
8926+
runner._mark_backfills_complete()
8927+
b = session.get(Backfill, backfill_id)
8928+
assert b.completed_at is not None
8929+
8930+
88808931
class Key1Mapper(CorePartitionMapper):
88818932
"""Partition Mapper that returns only key-1 as downstream key"""
88828933

0 commit comments

Comments
 (0)