Skip to content

Commit 37c43dd

Browse files
authored
Fix backfill marked complete before DagRuns are created (#62561)
1 parent 891c7fb commit 37c43dd

3 files changed

Lines changed: 269 additions & 2 deletions

File tree

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fix backfill marked complete before DagRuns are created; add age-based cleanup for orphaned backfills.

‎airflow-core/src/airflow/jobs/scheduler_job_runner.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@
7171
TaskInletAssetReference,
7272
TaskOutletAssetReference,
7373
)
74-
from airflow.models.backfill import Backfill
74+
from airflow.models.backfill import Backfill, BackfillDagRun
7575
from airflow.models.callback import Callback, CallbackType, ExecutorCallback
7676
from airflow.models.dag import DagModel
7777
from airflow.models.dag_version import DagVersion
@@ -1861,8 +1861,17 @@ def _mark_backfills_complete(self, session: Session = NEW_SESSION) -> None:
18611861
unfinished_states = (DagRunState.RUNNING, DagRunState.QUEUED)
18621862
now = timezone.utcnow()
18631863
# todo: AIP-78 simplify this function to an update statement
1864+
initializing_cutoff = now - timedelta(minutes=2)
18641865
query = select(Backfill).where(
18651866
Backfill.completed_at.is_(None),
1867+
# Guard: backfill must have at least one association,
1868+
# otherwise it is still being set up (see #61375).
1869+
# Allow cleanup of orphaned backfills older than 2 minutes
1870+
# that failed during initialization and never got any associations.
1871+
or_(
1872+
exists(select(BackfillDagRun.id).where(BackfillDagRun.backfill_id == Backfill.id)),
1873+
Backfill.created_at < initializing_cutoff,
1874+
),
18661875
~exists(
18671876
select(DagRun.id).where(
18681877
and_(DagRun.backfill_id == Backfill.id, DagRun.state.in_(unfinished_states))

‎airflow-core/tests/unit/jobs/test_scheduler_job.py‎

Lines changed: 258 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@
6565
AssetPartitionDagRun,
6666
PartitionedAssetKeyLog,
6767
)
68-
from airflow.models.backfill import Backfill, _create_backfill
68+
from airflow.models.backfill import Backfill, BackfillDagRun, ReprocessBehavior, _create_backfill
6969
from airflow.models.callback import ExecutorCallback
7070
from airflow.models.dag import DagModel, get_last_dagrun, infer_automated_data_interval
7171
from airflow.models.dag_version import DagVersion
@@ -8792,6 +8792,263 @@ def test_mark_backfills_completed(dag_maker, session):
87928792
assert b.completed_at.timestamp() > 0
87938793

87948794

8795+
def test_mark_backfills_complete_skips_initializing_backfill(dag_maker, session):
8796+
clear_db_backfills()
8797+
dag_id = "test_backfill_race_lifecycle"
8798+
with dag_maker(serialized=True, dag_id=dag_id, schedule="@daily"):
8799+
BashOperator(task_id="hi", bash_command="echo hi")
8800+
b = Backfill(
8801+
dag_id=dag_id,
8802+
from_date=pendulum.parse("2021-01-01"),
8803+
to_date=pendulum.parse("2021-01-03"),
8804+
max_active_runs=10,
8805+
dag_run_conf={},
8806+
reprocess_behavior=ReprocessBehavior.NONE,
8807+
)
8808+
session.add(b)
8809+
session.commit()
8810+
backfill_id = b.id
8811+
session.expunge_all()
8812+
runner = SchedulerJobRunner(
8813+
job=Job(job_type=SchedulerJobRunner.job_type), executors=[MockExecutor(do_update=False)]
8814+
)
8815+
runner._mark_backfills_complete()
8816+
b = session.get(Backfill, backfill_id)
8817+
assert b.completed_at is None
8818+
session.expunge_all()
8819+
dr = DagRun(
8820+
dag_id=dag_id,
8821+
run_id="backfill__2021-01-01T00:00:00+00:00",
8822+
run_type=DagRunType.BACKFILL_JOB,
8823+
logical_date=pendulum.parse("2021-01-01"),
8824+
data_interval=(pendulum.parse("2021-01-01"), pendulum.parse("2021-01-02")),
8825+
run_after=pendulum.parse("2021-01-02"),
8826+
state=DagRunState.SUCCESS,
8827+
backfill_id=backfill_id,
8828+
)
8829+
session.add(dr)
8830+
session.flush()
8831+
session.add(
8832+
BackfillDagRun(
8833+
backfill_id=backfill_id,
8834+
dag_run_id=dr.id,
8835+
logical_date=pendulum.parse("2021-01-01"),
8836+
sort_ordinal=1,
8837+
)
8838+
)
8839+
session.commit()
8840+
session.expunge_all()
8841+
runner._mark_backfills_complete()
8842+
b = session.get(Backfill, backfill_id)
8843+
assert b.completed_at is not None
8844+
8845+
8846+
def test_mark_backfills_complete_cleans_orphan_after_cutoff(dag_maker, session):
8847+
"""Backfill with no BackfillDagRun rows older than 2 minutes should be auto-completed."""
8848+
clear_db_backfills()
8849+
dag_id = "test_backfill_orphan_cleanup"
8850+
with dag_maker(serialized=True, dag_id=dag_id, schedule="@daily"):
8851+
BashOperator(task_id="hi", bash_command="echo hi")
8852+
b = Backfill(
8853+
dag_id=dag_id,
8854+
from_date=pendulum.parse("2021-01-01"),
8855+
to_date=pendulum.parse("2021-01-03"),
8856+
max_active_runs=10,
8857+
dag_run_conf={},
8858+
reprocess_behavior=ReprocessBehavior.NONE,
8859+
)
8860+
session.add(b)
8861+
session.commit()
8862+
backfill_id = b.id
8863+
session.expunge_all()
8864+
# Travel 3 minutes into the future so the backfill is past the 2-minute cutoff
8865+
with time_machine.travel(pendulum.now("UTC").add(minutes=3), tick=False):
8866+
runner = SchedulerJobRunner(
8867+
job=Job(job_type=SchedulerJobRunner.job_type), executors=[MockExecutor(do_update=False)]
8868+
)
8869+
runner._mark_backfills_complete()
8870+
b = session.get(Backfill, backfill_id)
8871+
assert b.completed_at is not None
8872+
8873+
8874+
def test_mark_backfills_complete_keeps_old_backfill_with_running_dagruns(dag_maker, session):
8875+
"""Old backfill (>2 min) with running DagRuns must NOT be marked complete."""
8876+
clear_db_backfills()
8877+
dag_id = "test_backfill_old_with_runs"
8878+
with dag_maker(serialized=True, dag_id=dag_id, schedule="@daily"):
8879+
BashOperator(task_id="hi", bash_command="echo hi")
8880+
b = Backfill(
8881+
dag_id=dag_id,
8882+
from_date=pendulum.parse("2021-01-01"),
8883+
to_date=pendulum.parse("2021-01-03"),
8884+
max_active_runs=10,
8885+
dag_run_conf={},
8886+
reprocess_behavior=ReprocessBehavior.NONE,
8887+
)
8888+
session.add(b)
8889+
session.commit()
8890+
backfill_id = b.id
8891+
dr = DagRun(
8892+
dag_id=dag_id,
8893+
run_id="backfill__2021-01-01T00:00:00+00:00",
8894+
run_type=DagRunType.BACKFILL_JOB,
8895+
logical_date=pendulum.parse("2021-01-01"),
8896+
data_interval=(pendulum.parse("2021-01-01"), pendulum.parse("2021-01-02")),
8897+
run_after=pendulum.parse("2021-01-02"),
8898+
state=DagRunState.RUNNING,
8899+
backfill_id=backfill_id,
8900+
)
8901+
session.add(dr)
8902+
session.flush()
8903+
session.add(
8904+
BackfillDagRun(
8905+
backfill_id=backfill_id,
8906+
dag_run_id=dr.id,
8907+
logical_date=pendulum.parse("2021-01-01"),
8908+
sort_ordinal=1,
8909+
)
8910+
)
8911+
session.commit()
8912+
session.expunge_all()
8913+
# Travel 3 minutes into the future; backfill is old but has a RUNNING DagRun
8914+
with time_machine.travel(pendulum.now("UTC").add(minutes=3), tick=False):
8915+
runner = SchedulerJobRunner(
8916+
job=Job(job_type=SchedulerJobRunner.job_type), executors=[MockExecutor(do_update=False)]
8917+
)
8918+
runner._mark_backfills_complete()
8919+
b = session.get(Backfill, backfill_id)
8920+
assert b.completed_at is None
8921+
8922+
8923+
def test_mark_backfills_complete_young_backfill_with_finished_runs(dag_maker, session):
8924+
"""Young backfill (<2 min) with all SUCCESS DagRuns completes immediately."""
8925+
clear_db_backfills()
8926+
dag_id = "test_backfill_young_finished"
8927+
with dag_maker(serialized=True, dag_id=dag_id, schedule="@daily"):
8928+
BashOperator(task_id="hi", bash_command="echo hi")
8929+
b = Backfill(
8930+
dag_id=dag_id,
8931+
from_date=pendulum.parse("2021-01-01"),
8932+
to_date=pendulum.parse("2021-01-03"),
8933+
max_active_runs=10,
8934+
dag_run_conf={},
8935+
reprocess_behavior=ReprocessBehavior.NONE,
8936+
)
8937+
session.add(b)
8938+
session.commit()
8939+
backfill_id = b.id
8940+
dr = DagRun(
8941+
dag_id=dag_id,
8942+
run_id="backfill__2021-01-01T00:00:00+00:00",
8943+
run_type=DagRunType.BACKFILL_JOB,
8944+
logical_date=pendulum.parse("2021-01-01"),
8945+
data_interval=(pendulum.parse("2021-01-01"), pendulum.parse("2021-01-02")),
8946+
run_after=pendulum.parse("2021-01-02"),
8947+
state=DagRunState.SUCCESS,
8948+
backfill_id=backfill_id,
8949+
)
8950+
session.add(dr)
8951+
session.flush()
8952+
session.add(
8953+
BackfillDagRun(
8954+
backfill_id=backfill_id,
8955+
dag_run_id=dr.id,
8956+
logical_date=pendulum.parse("2021-01-01"),
8957+
sort_ordinal=1,
8958+
)
8959+
)
8960+
session.commit()
8961+
session.expunge_all()
8962+
# No time travel — backfill was just created, should still complete
8963+
runner = SchedulerJobRunner(
8964+
job=Job(job_type=SchedulerJobRunner.job_type), executors=[MockExecutor(do_update=False)]
8965+
)
8966+
runner._mark_backfills_complete()
8967+
b = session.get(Backfill, backfill_id)
8968+
assert b.completed_at is not None
8969+
8970+
8971+
def test_mark_backfills_complete_multiple_independent(dag_maker, session):
8972+
"""Two backfills: one finished, one running — only the finished one completes."""
8973+
clear_db_backfills()
8974+
with dag_maker(serialized=True, dag_id="dag_finished", schedule="@daily"):
8975+
BashOperator(task_id="hi", bash_command="echo hi")
8976+
with dag_maker(serialized=True, dag_id="dag_running", schedule="@daily"):
8977+
BashOperator(task_id="hi", bash_command="echo hi")
8978+
b_finished = Backfill(
8979+
dag_id="dag_finished",
8980+
from_date=pendulum.parse("2021-01-01"),
8981+
to_date=pendulum.parse("2021-01-03"),
8982+
max_active_runs=10,
8983+
dag_run_conf={},
8984+
reprocess_behavior=ReprocessBehavior.NONE,
8985+
)
8986+
b_running = Backfill(
8987+
dag_id="dag_running",
8988+
from_date=pendulum.parse("2021-01-01"),
8989+
to_date=pendulum.parse("2021-01-03"),
8990+
max_active_runs=10,
8991+
dag_run_conf={},
8992+
reprocess_behavior=ReprocessBehavior.NONE,
8993+
)
8994+
session.add_all([b_finished, b_running])
8995+
session.commit()
8996+
finished_id = b_finished.id
8997+
running_id = b_running.id
8998+
# Finished backfill: SUCCESS DagRun
8999+
dr1 = DagRun(
9000+
dag_id="dag_finished",
9001+
run_id="backfill__2021-01-01T00:00:00+00:00",
9002+
run_type=DagRunType.BACKFILL_JOB,
9003+
logical_date=pendulum.parse("2021-01-01"),
9004+
data_interval=(pendulum.parse("2021-01-01"), pendulum.parse("2021-01-02")),
9005+
run_after=pendulum.parse("2021-01-02"),
9006+
state=DagRunState.SUCCESS,
9007+
backfill_id=finished_id,
9008+
)
9009+
session.add(dr1)
9010+
session.flush()
9011+
session.add(
9012+
BackfillDagRun(
9013+
backfill_id=finished_id,
9014+
dag_run_id=dr1.id,
9015+
logical_date=pendulum.parse("2021-01-01"),
9016+
sort_ordinal=1,
9017+
)
9018+
)
9019+
# Running backfill: RUNNING DagRun
9020+
dr2 = DagRun(
9021+
dag_id="dag_running",
9022+
run_id="backfill__2021-01-01T00:00:00+00:00",
9023+
run_type=DagRunType.BACKFILL_JOB,
9024+
logical_date=pendulum.parse("2021-01-01"),
9025+
data_interval=(pendulum.parse("2021-01-01"), pendulum.parse("2021-01-02")),
9026+
run_after=pendulum.parse("2021-01-02"),
9027+
state=DagRunState.RUNNING,
9028+
backfill_id=running_id,
9029+
)
9030+
session.add(dr2)
9031+
session.flush()
9032+
session.add(
9033+
BackfillDagRun(
9034+
backfill_id=running_id,
9035+
dag_run_id=dr2.id,
9036+
logical_date=pendulum.parse("2021-01-01"),
9037+
sort_ordinal=1,
9038+
)
9039+
)
9040+
session.commit()
9041+
session.expunge_all()
9042+
runner = SchedulerJobRunner(
9043+
job=Job(job_type=SchedulerJobRunner.job_type), executors=[MockExecutor(do_update=False)]
9044+
)
9045+
runner._mark_backfills_complete()
9046+
b_finished = session.get(Backfill, finished_id)
9047+
b_running = session.get(Backfill, running_id)
9048+
assert b_finished.completed_at is not None
9049+
assert b_running.completed_at is None
9050+
9051+
87959052
class Key1Mapper(CorePartitionMapper):
87969053
"""Partition Mapper that returns only key-1 as downstream key"""
87979054

0 commit comments

Comments
 (0)