|
21 | 21 | from collections import defaultdict |
22 | 22 | from collections.abc import Mapping |
23 | 23 | from concurrent.futures import ThreadPoolExecutor |
24 | | -from contextlib import contextmanager |
| 24 | +from contextlib import contextmanager, nullcontext |
25 | 25 | from functools import partial, reduce |
26 | 26 | from typing import TYPE_CHECKING |
27 | 27 | from unittest import mock |
@@ -1954,30 +1954,150 @@ def test_dagrun_deadline_does_not_warn_for_average_runtime_without_history( |
1954 | 1954 | "log_level": "warning", |
1955 | 1955 | } not in caplog |
1956 | 1956 |
|
| 1957 | + @pytest.mark.parametrize( |
| 1958 | + ("interval", "failure"), |
| 1959 | + [ |
| 1960 | + pytest.param( |
| 1961 | + VariableInterval("missing_key"), |
| 1962 | + mock.patch.object(Variable, "get", side_effect=KeyError), |
| 1963 | + id="unresolvable_interval", |
| 1964 | + ), |
| 1965 | + pytest.param( |
| 1966 | + datetime.timedelta(hours=1), |
| 1967 | + mock.patch( |
| 1968 | + "airflow.serialization.definitions.dag.decode_deadline_alert", |
| 1969 | + autospec=True, |
| 1970 | + side_effect=ValueError("corrupt deadline alert blob"), |
| 1971 | + ), |
| 1972 | + id="undecodable_alert", |
| 1973 | + ), |
| 1974 | + pytest.param( |
| 1975 | + datetime.timedelta(hours=1), |
| 1976 | + mock.patch.object( |
| 1977 | + SerializedReferenceModels.FixedDatetimeDeadline, |
| 1978 | + "evaluate_with", |
| 1979 | + autospec=True, |
| 1980 | + side_effect=RuntimeError("evaluate_with failed"), |
| 1981 | + ), |
| 1982 | + id="unevaluable_reference", |
| 1983 | + ), |
| 1984 | + ], |
| 1985 | + ) |
| 1986 | + @mock.patch("airflow._shared.observability.metrics.stats.incr") |
1957 | 1987 | @mock.patch.object(Deadline, "prune_deadlines") |
1958 | | - def test_dagrun_deadline_variable_interval_missing_variable_fails(self, _, session, deadline_test_dag): |
| 1988 | + def test_dagrun_deadline_failure_does_not_abort_dagrun( |
| 1989 | + self, _, mock_stats_incr, interval, failure, session, deadline_test_dag |
| 1990 | + ): |
| 1991 | + future_date = datetime.datetime(2037, 1, 1, tzinfo=datetime.UTC) |
1959 | 1992 |
|
1960 | | - with mock.patch.object( |
1961 | | - Variable, |
1962 | | - "get", |
1963 | | - side_effect=KeyError, |
| 1993 | + scheduler_dag = deadline_test_dag( |
| 1994 | + deadline=DeadlineAlert( |
| 1995 | + reference=DeadlineReference.FIXED_DATETIME(future_date), |
| 1996 | + interval=interval, |
| 1997 | + callback=AsyncCallback(empty_callback_for_deadline), |
| 1998 | + ), |
| 1999 | + ) |
| 2000 | + |
| 2001 | + with ( |
| 2002 | + conf_vars({("core", "multi_team"): "true"}), |
| 2003 | + mock.patch("airflow.models.dag.DagModel.get_team_name", return_value="team_alpha"), |
| 2004 | + failure, |
1964 | 2005 | ): |
1965 | | - future_date = datetime.datetime.now() + datetime.timedelta(days=365) |
| 2006 | + dag_run = self.create_dag_run( |
| 2007 | + dag=scheduler_dag, |
| 2008 | + task_states={"task_1": TaskInstanceState.SUCCESS}, |
| 2009 | + session=session, |
| 2010 | + ) |
| 2011 | + |
| 2012 | + assert dag_run is not None |
| 2013 | + assert session.execute(select(Deadline)).scalars().one_or_none() is None |
| 2014 | + mock_stats_incr.assert_any_call( |
| 2015 | + "deadline_alerts.deadline_creation_failed", |
| 2016 | + tags={"dag_id": scheduler_dag.dag_id, "team_name": "team_alpha"}, |
| 2017 | + ) |
1966 | 2018 |
|
1967 | | - scheduler_dag = deadline_test_dag( |
1968 | | - deadline=DeadlineAlert( |
1969 | | - reference=DeadlineReference.FIXED_DATETIME(future_date), |
1970 | | - interval=VariableInterval("missing_key"), |
1971 | | - callback=AsyncCallback(empty_callback_for_deadline), |
| 2019 | + @mock.patch("airflow._shared.observability.metrics.stats.incr") |
| 2020 | + @mock.patch.object(Deadline, "prune_deadlines") |
| 2021 | + @mock.patch.object(Variable, "get") |
| 2022 | + def test_dagrun_deadline_failure_that_detaches_orm_objects_still_skips( |
| 2023 | + self, mock_variable_get, _, mock_stats_incr, session, deadline_test_dag |
| 2024 | + ): |
| 2025 | + """A failing interval resolution can roll back and close the caller's session underneath |
| 2026 | + the handler (``create_session`` reusing the scoped session does exactly that), detaching |
| 2027 | + every ORM instance. The skip path must still log and count without touching ORM state.""" |
| 2028 | + scheduler_dag = deadline_test_dag( |
| 2029 | + deadline=DeadlineAlert( |
| 2030 | + reference=DeadlineReference.FIXED_DATETIME( |
| 2031 | + datetime.datetime(2037, 1, 1, tzinfo=datetime.UTC) |
1972 | 2032 | ), |
1973 | | - ) |
| 2033 | + interval=VariableInterval("missing_key"), |
| 2034 | + callback=AsyncCallback(empty_callback_for_deadline), |
| 2035 | + ), |
| 2036 | + ) |
| 2037 | + # The alert rows must be persistent from an earlier transaction (as in the live |
| 2038 | + # scheduler, where the dag processor committed them) so the rollback below expires |
| 2039 | + # them and the close detaches them instead of returning them to transient. |
| 2040 | + session.commit() |
1974 | 2041 |
|
1975 | | - with pytest.raises(ValueError, match="not found"): |
1976 | | - self.create_dag_run( |
1977 | | - dag=scheduler_dag, |
1978 | | - task_states={"task_1": TaskInstanceState.SUCCESS}, |
1979 | | - session=session, |
1980 | | - ) |
| 2042 | + def teardown_session_and_raise(*args, **kwargs): |
| 2043 | + session.rollback() |
| 2044 | + session.close() |
| 2045 | + raise KeyError("missing_key") |
| 2046 | + |
| 2047 | + mock_variable_get.side_effect = teardown_session_and_raise |
| 2048 | + |
| 2049 | + dag_run = self.create_dag_run(dag=scheduler_dag, session=session) |
| 2050 | + |
| 2051 | + assert dag_run is not None |
| 2052 | + mock_stats_incr.assert_any_call( |
| 2053 | + "deadline_alerts.deadline_creation_failed", |
| 2054 | + tags={"dag_id": scheduler_dag.dag_id}, |
| 2055 | + ) |
| 2056 | + |
| 2057 | + @pytest.mark.parametrize( |
| 2058 | + ("interval", "expect_deadline", "variable_value"), |
| 2059 | + [ |
| 2060 | + pytest.param(datetime.timedelta(seconds=5), True, None, id="deadline_created"), |
| 2061 | + pytest.param(VariableInterval("missing_key"), False, None, id="deadline_skipped"), |
| 2062 | + pytest.param(VariableInterval("resolvable_key"), True, "4200", id="variable_interval_resolved"), |
| 2063 | + ], |
| 2064 | + ) |
| 2065 | + @mock.patch.object(Deadline, "prune_deadlines") |
| 2066 | + def test_dagrun_deadline_handling_does_not_commit( |
| 2067 | + self, _, interval, expect_deadline, variable_value, session, deadline_test_dag |
| 2068 | + ): |
| 2069 | + """DagRuns are created under ``prohibit_commit``, so creating a deadline, skipping one whose |
| 2070 | + interval will not resolve, and resolving a ``VariableInterval`` out of the metadata database |
| 2071 | + all have to leave the scheduler's session uncommitted.""" |
| 2072 | + future_date = datetime.datetime(2037, 1, 1, tzinfo=datetime.UTC) |
| 2073 | + scheduler_dag = deadline_test_dag( |
| 2074 | + deadline=DeadlineAlert( |
| 2075 | + reference=DeadlineReference.FIXED_DATETIME(future_date), |
| 2076 | + interval=interval, |
| 2077 | + callback=AsyncCallback(empty_callback_for_deadline), |
| 2078 | + ), |
| 2079 | + ) |
| 2080 | + |
| 2081 | + if variable_value is None: |
| 2082 | + variable_lookup = mock.patch.object(Variable, "get", side_effect=KeyError) |
| 2083 | + else: |
| 2084 | + # A real row committed before the guard opens, so resolve() has to go through |
| 2085 | + # MetastoreBackend. Drop the session forwarding and the backend opens the same scoped |
| 2086 | + # session and commits it, which the guard rejects and the lookup then reports as missing. |
| 2087 | + Variable.set(key=interval.key, value=variable_value, session=session) |
| 2088 | + session.commit() |
| 2089 | + variable_lookup = nullcontext() |
| 2090 | + |
| 2091 | + # No task_states: the helper's get_task_instance() opens its own session and would trip the guard. |
| 2092 | + with prohibit_commit(session), variable_lookup: |
| 2093 | + dag_run = self.create_dag_run(dag=scheduler_dag, session=session) |
| 2094 | + session.flush() |
| 2095 | + |
| 2096 | + assert dag_run is not None |
| 2097 | + deadline = session.execute(select(Deadline)).scalars().one_or_none() |
| 2098 | + assert (deadline is not None) is expect_deadline |
| 2099 | + if variable_value is not None: |
| 2100 | + assert deadline.deadline_time == future_date + datetime.timedelta(seconds=int(variable_value)) |
1981 | 2101 |
|
1982 | 2102 |
|
1983 | 2103 | @pytest.mark.parametrize( |
|
0 commit comments