|
65 | 65 | AssetPartitionDagRun, |
66 | 66 | PartitionedAssetKeyLog, |
67 | 67 | ) |
68 | | -from airflow.models.backfill import Backfill, _create_backfill |
| 68 | +from airflow.models.backfill import Backfill, BackfillDagRun, ReprocessBehavior, _create_backfill |
69 | 69 | from airflow.models.callback import ExecutorCallback |
70 | 70 | from airflow.models.dag import DagModel, get_last_dagrun, infer_automated_data_interval |
71 | 71 | from airflow.models.dag_version import DagVersion |
@@ -8792,6 +8792,263 @@ def test_mark_backfills_completed(dag_maker, session): |
8792 | 8792 | assert b.completed_at.timestamp() > 0 |
8793 | 8793 |
|
8794 | 8794 |
|
| 8795 | +def test_mark_backfills_complete_skips_initializing_backfill(dag_maker, session): |
| 8796 | + clear_db_backfills() |
| 8797 | + dag_id = "test_backfill_race_lifecycle" |
| 8798 | + with dag_maker(serialized=True, dag_id=dag_id, schedule="@daily"): |
| 8799 | + BashOperator(task_id="hi", bash_command="echo hi") |
| 8800 | + b = Backfill( |
| 8801 | + dag_id=dag_id, |
| 8802 | + from_date=pendulum.parse("2021-01-01"), |
| 8803 | + to_date=pendulum.parse("2021-01-03"), |
| 8804 | + max_active_runs=10, |
| 8805 | + dag_run_conf={}, |
| 8806 | + reprocess_behavior=ReprocessBehavior.NONE, |
| 8807 | + ) |
| 8808 | + session.add(b) |
| 8809 | + session.commit() |
| 8810 | + backfill_id = b.id |
| 8811 | + session.expunge_all() |
| 8812 | + runner = SchedulerJobRunner( |
| 8813 | + job=Job(job_type=SchedulerJobRunner.job_type), executors=[MockExecutor(do_update=False)] |
| 8814 | + ) |
| 8815 | + runner._mark_backfills_complete() |
| 8816 | + b = session.get(Backfill, backfill_id) |
| 8817 | + assert b.completed_at is None |
| 8818 | + session.expunge_all() |
| 8819 | + dr = DagRun( |
| 8820 | + dag_id=dag_id, |
| 8821 | + run_id="backfill__2021-01-01T00:00:00+00:00", |
| 8822 | + run_type=DagRunType.BACKFILL_JOB, |
| 8823 | + logical_date=pendulum.parse("2021-01-01"), |
| 8824 | + data_interval=(pendulum.parse("2021-01-01"), pendulum.parse("2021-01-02")), |
| 8825 | + run_after=pendulum.parse("2021-01-02"), |
| 8826 | + state=DagRunState.SUCCESS, |
| 8827 | + backfill_id=backfill_id, |
| 8828 | + ) |
| 8829 | + session.add(dr) |
| 8830 | + session.flush() |
| 8831 | + session.add( |
| 8832 | + BackfillDagRun( |
| 8833 | + backfill_id=backfill_id, |
| 8834 | + dag_run_id=dr.id, |
| 8835 | + logical_date=pendulum.parse("2021-01-01"), |
| 8836 | + sort_ordinal=1, |
| 8837 | + ) |
| 8838 | + ) |
| 8839 | + session.commit() |
| 8840 | + session.expunge_all() |
| 8841 | + runner._mark_backfills_complete() |
| 8842 | + b = session.get(Backfill, backfill_id) |
| 8843 | + assert b.completed_at is not None |
| 8844 | + |
| 8845 | + |
| 8846 | +def test_mark_backfills_complete_cleans_orphan_after_cutoff(dag_maker, session): |
| 8847 | + """Backfill with no BackfillDagRun rows older than 2 minutes should be auto-completed.""" |
| 8848 | + clear_db_backfills() |
| 8849 | + dag_id = "test_backfill_orphan_cleanup" |
| 8850 | + with dag_maker(serialized=True, dag_id=dag_id, schedule="@daily"): |
| 8851 | + BashOperator(task_id="hi", bash_command="echo hi") |
| 8852 | + b = Backfill( |
| 8853 | + dag_id=dag_id, |
| 8854 | + from_date=pendulum.parse("2021-01-01"), |
| 8855 | + to_date=pendulum.parse("2021-01-03"), |
| 8856 | + max_active_runs=10, |
| 8857 | + dag_run_conf={}, |
| 8858 | + reprocess_behavior=ReprocessBehavior.NONE, |
| 8859 | + ) |
| 8860 | + session.add(b) |
| 8861 | + session.commit() |
| 8862 | + backfill_id = b.id |
| 8863 | + session.expunge_all() |
| 8864 | + # Travel 3 minutes into the future so the backfill is past the 2-minute cutoff |
| 8865 | + with time_machine.travel(pendulum.now("UTC").add(minutes=3), tick=False): |
| 8866 | + runner = SchedulerJobRunner( |
| 8867 | + job=Job(job_type=SchedulerJobRunner.job_type), executors=[MockExecutor(do_update=False)] |
| 8868 | + ) |
| 8869 | + runner._mark_backfills_complete() |
| 8870 | + b = session.get(Backfill, backfill_id) |
| 8871 | + assert b.completed_at is not None |
| 8872 | + |
| 8873 | + |
| 8874 | +def test_mark_backfills_complete_keeps_old_backfill_with_running_dagruns(dag_maker, session): |
| 8875 | + """Old backfill (>2 min) with running DagRuns must NOT be marked complete.""" |
| 8876 | + clear_db_backfills() |
| 8877 | + dag_id = "test_backfill_old_with_runs" |
| 8878 | + with dag_maker(serialized=True, dag_id=dag_id, schedule="@daily"): |
| 8879 | + BashOperator(task_id="hi", bash_command="echo hi") |
| 8880 | + b = Backfill( |
| 8881 | + dag_id=dag_id, |
| 8882 | + from_date=pendulum.parse("2021-01-01"), |
| 8883 | + to_date=pendulum.parse("2021-01-03"), |
| 8884 | + max_active_runs=10, |
| 8885 | + dag_run_conf={}, |
| 8886 | + reprocess_behavior=ReprocessBehavior.NONE, |
| 8887 | + ) |
| 8888 | + session.add(b) |
| 8889 | + session.commit() |
| 8890 | + backfill_id = b.id |
| 8891 | + dr = DagRun( |
| 8892 | + dag_id=dag_id, |
| 8893 | + run_id="backfill__2021-01-01T00:00:00+00:00", |
| 8894 | + run_type=DagRunType.BACKFILL_JOB, |
| 8895 | + logical_date=pendulum.parse("2021-01-01"), |
| 8896 | + data_interval=(pendulum.parse("2021-01-01"), pendulum.parse("2021-01-02")), |
| 8897 | + run_after=pendulum.parse("2021-01-02"), |
| 8898 | + state=DagRunState.RUNNING, |
| 8899 | + backfill_id=backfill_id, |
| 8900 | + ) |
| 8901 | + session.add(dr) |
| 8902 | + session.flush() |
| 8903 | + session.add( |
| 8904 | + BackfillDagRun( |
| 8905 | + backfill_id=backfill_id, |
| 8906 | + dag_run_id=dr.id, |
| 8907 | + logical_date=pendulum.parse("2021-01-01"), |
| 8908 | + sort_ordinal=1, |
| 8909 | + ) |
| 8910 | + ) |
| 8911 | + session.commit() |
| 8912 | + session.expunge_all() |
| 8913 | + # Travel 3 minutes into the future; backfill is old but has a RUNNING DagRun |
| 8914 | + with time_machine.travel(pendulum.now("UTC").add(minutes=3), tick=False): |
| 8915 | + runner = SchedulerJobRunner( |
| 8916 | + job=Job(job_type=SchedulerJobRunner.job_type), executors=[MockExecutor(do_update=False)] |
| 8917 | + ) |
| 8918 | + runner._mark_backfills_complete() |
| 8919 | + b = session.get(Backfill, backfill_id) |
| 8920 | + assert b.completed_at is None |
| 8921 | + |
| 8922 | + |
| 8923 | +def test_mark_backfills_complete_young_backfill_with_finished_runs(dag_maker, session): |
| 8924 | + """Young backfill (<2 min) with all SUCCESS DagRuns completes immediately.""" |
| 8925 | + clear_db_backfills() |
| 8926 | + dag_id = "test_backfill_young_finished" |
| 8927 | + with dag_maker(serialized=True, dag_id=dag_id, schedule="@daily"): |
| 8928 | + BashOperator(task_id="hi", bash_command="echo hi") |
| 8929 | + b = Backfill( |
| 8930 | + dag_id=dag_id, |
| 8931 | + from_date=pendulum.parse("2021-01-01"), |
| 8932 | + to_date=pendulum.parse("2021-01-03"), |
| 8933 | + max_active_runs=10, |
| 8934 | + dag_run_conf={}, |
| 8935 | + reprocess_behavior=ReprocessBehavior.NONE, |
| 8936 | + ) |
| 8937 | + session.add(b) |
| 8938 | + session.commit() |
| 8939 | + backfill_id = b.id |
| 8940 | + dr = DagRun( |
| 8941 | + dag_id=dag_id, |
| 8942 | + run_id="backfill__2021-01-01T00:00:00+00:00", |
| 8943 | + run_type=DagRunType.BACKFILL_JOB, |
| 8944 | + logical_date=pendulum.parse("2021-01-01"), |
| 8945 | + data_interval=(pendulum.parse("2021-01-01"), pendulum.parse("2021-01-02")), |
| 8946 | + run_after=pendulum.parse("2021-01-02"), |
| 8947 | + state=DagRunState.SUCCESS, |
| 8948 | + backfill_id=backfill_id, |
| 8949 | + ) |
| 8950 | + session.add(dr) |
| 8951 | + session.flush() |
| 8952 | + session.add( |
| 8953 | + BackfillDagRun( |
| 8954 | + backfill_id=backfill_id, |
| 8955 | + dag_run_id=dr.id, |
| 8956 | + logical_date=pendulum.parse("2021-01-01"), |
| 8957 | + sort_ordinal=1, |
| 8958 | + ) |
| 8959 | + ) |
| 8960 | + session.commit() |
| 8961 | + session.expunge_all() |
| 8962 | + # No time travel — backfill was just created, should still complete |
| 8963 | + runner = SchedulerJobRunner( |
| 8964 | + job=Job(job_type=SchedulerJobRunner.job_type), executors=[MockExecutor(do_update=False)] |
| 8965 | + ) |
| 8966 | + runner._mark_backfills_complete() |
| 8967 | + b = session.get(Backfill, backfill_id) |
| 8968 | + assert b.completed_at is not None |
| 8969 | + |
| 8970 | + |
| 8971 | +def test_mark_backfills_complete_multiple_independent(dag_maker, session): |
| 8972 | + """Two backfills: one finished, one running — only the finished one completes.""" |
| 8973 | + clear_db_backfills() |
| 8974 | + with dag_maker(serialized=True, dag_id="dag_finished", schedule="@daily"): |
| 8975 | + BashOperator(task_id="hi", bash_command="echo hi") |
| 8976 | + with dag_maker(serialized=True, dag_id="dag_running", schedule="@daily"): |
| 8977 | + BashOperator(task_id="hi", bash_command="echo hi") |
| 8978 | + b_finished = Backfill( |
| 8979 | + dag_id="dag_finished", |
| 8980 | + from_date=pendulum.parse("2021-01-01"), |
| 8981 | + to_date=pendulum.parse("2021-01-03"), |
| 8982 | + max_active_runs=10, |
| 8983 | + dag_run_conf={}, |
| 8984 | + reprocess_behavior=ReprocessBehavior.NONE, |
| 8985 | + ) |
| 8986 | + b_running = Backfill( |
| 8987 | + dag_id="dag_running", |
| 8988 | + from_date=pendulum.parse("2021-01-01"), |
| 8989 | + to_date=pendulum.parse("2021-01-03"), |
| 8990 | + max_active_runs=10, |
| 8991 | + dag_run_conf={}, |
| 8992 | + reprocess_behavior=ReprocessBehavior.NONE, |
| 8993 | + ) |
| 8994 | + session.add_all([b_finished, b_running]) |
| 8995 | + session.commit() |
| 8996 | + finished_id = b_finished.id |
| 8997 | + running_id = b_running.id |
| 8998 | + # Finished backfill: SUCCESS DagRun |
| 8999 | + dr1 = DagRun( |
| 9000 | + dag_id="dag_finished", |
| 9001 | + run_id="backfill__2021-01-01T00:00:00+00:00", |
| 9002 | + run_type=DagRunType.BACKFILL_JOB, |
| 9003 | + logical_date=pendulum.parse("2021-01-01"), |
| 9004 | + data_interval=(pendulum.parse("2021-01-01"), pendulum.parse("2021-01-02")), |
| 9005 | + run_after=pendulum.parse("2021-01-02"), |
| 9006 | + state=DagRunState.SUCCESS, |
| 9007 | + backfill_id=finished_id, |
| 9008 | + ) |
| 9009 | + session.add(dr1) |
| 9010 | + session.flush() |
| 9011 | + session.add( |
| 9012 | + BackfillDagRun( |
| 9013 | + backfill_id=finished_id, |
| 9014 | + dag_run_id=dr1.id, |
| 9015 | + logical_date=pendulum.parse("2021-01-01"), |
| 9016 | + sort_ordinal=1, |
| 9017 | + ) |
| 9018 | + ) |
| 9019 | + # Running backfill: RUNNING DagRun |
| 9020 | + dr2 = DagRun( |
| 9021 | + dag_id="dag_running", |
| 9022 | + run_id="backfill__2021-01-01T00:00:00+00:00", |
| 9023 | + run_type=DagRunType.BACKFILL_JOB, |
| 9024 | + logical_date=pendulum.parse("2021-01-01"), |
| 9025 | + data_interval=(pendulum.parse("2021-01-01"), pendulum.parse("2021-01-02")), |
| 9026 | + run_after=pendulum.parse("2021-01-02"), |
| 9027 | + state=DagRunState.RUNNING, |
| 9028 | + backfill_id=running_id, |
| 9029 | + ) |
| 9030 | + session.add(dr2) |
| 9031 | + session.flush() |
| 9032 | + session.add( |
| 9033 | + BackfillDagRun( |
| 9034 | + backfill_id=running_id, |
| 9035 | + dag_run_id=dr2.id, |
| 9036 | + logical_date=pendulum.parse("2021-01-01"), |
| 9037 | + sort_ordinal=1, |
| 9038 | + ) |
| 9039 | + ) |
| 9040 | + session.commit() |
| 9041 | + session.expunge_all() |
| 9042 | + runner = SchedulerJobRunner( |
| 9043 | + job=Job(job_type=SchedulerJobRunner.job_type), executors=[MockExecutor(do_update=False)] |
| 9044 | + ) |
| 9045 | + runner._mark_backfills_complete() |
| 9046 | + b_finished = session.get(Backfill, finished_id) |
| 9047 | + b_running = session.get(Backfill, running_id) |
| 9048 | + assert b_finished.completed_at is not None |
| 9049 | + assert b_running.completed_at is None |
| 9050 | + |
| 9051 | + |
8795 | 9052 | class Key1Mapper(CorePartitionMapper): |
8796 | 9053 | """Partition Mapper that returns only key-1 as downstream key""" |
8797 | 9054 |
|
|
0 commit comments