Skip to content

Commit f792bd6

Browse files
authored
Do not abort Dag run creation when a deadline alert fails (#68917)
* Do not abort Dag run creation when a deadline alert fails Deadline alerts are an observability feature layered on top of a Dag run, so they must never be able to stop one from existing. Decoding a stored alert, resolving its interval and evaluating its reference all happen inside create_dagrun, and any of them could raise straight out through Dag run creation -- so one malformed or unresolvable alert took the whole Dag run down with it, and on a Dag with several alerts the first failure starved the rest of their deadlines. The new deadline_alerts.deadline_creation_failed counter keeps these skips visible rather than silent.
1 parent 47d929c commit f792bd6

7 files changed

Lines changed: 237 additions & 65 deletions

File tree

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
A deadline alert that cannot be decoded, or whose interval or reference fails to resolve, no longer aborts creation of the Dag run it belongs to. The failing alert is logged and skipped, the remaining alerts still get their deadlines, and a new ``deadline_alerts.deadline_creation_failed`` counter is emitted for it.

‎airflow-core/src/airflow/serialization/definitions/dag.py‎

Lines changed: 78 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,13 @@
5050
from airflow.models.deadline_alert import DeadlineAlert as DeadlineAlertModel
5151
from airflow.models.task_coordinates import public_map_index_expression
5252
from airflow.models.tasklog import LogTemplate
53-
from airflow.serialization.decoders import decode_deadline_alert_model, resolve_deadline_alert_interval
54-
from airflow.serialization.definitions.deadline import SerializedReferenceModels
53+
from airflow.serialization.decoders import decode_deadline_alert, resolve_deadline_alert_interval
54+
from airflow.serialization.definitions.deadline import DeadlineAlertFields, SerializedReferenceModels
5555
from airflow.serialization.definitions.param import SerializedParamsDict
5656
from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction
5757
from airflow.utils.helpers import prune_dict
5858
from airflow.utils.session import NEW_SESSION, provide_session
59+
from airflow.utils.sqlalchemy import PROHIBIT_COMMIT_ERROR_MESSAGE
5960
from airflow.utils.state import DagRunState, TaskInstanceState
6061
from airflow.utils.types import DagRunType
6162

@@ -754,54 +755,87 @@ def _process_dagrun_deadline_alerts(
754755
select(DeadlineAlertModel).where(DeadlineAlertModel.serialized_dag_id == serialized_dag_id)
755756
).all()
756757

757-
for deadline_alert in deadline_alert_records:
758-
if not deadline_alert:
759-
continue
760-
761-
deserialized_deadline_alert = decode_deadline_alert_model(deadline_alert)
762-
interval = resolve_deadline_alert_interval(deserialized_deadline_alert, session=session)
758+
if not deadline_alert_records:
759+
return
763760

764-
if isinstance(deserialized_deadline_alert.reference, SerializedReferenceModels.TYPES.DAGRUN):
765-
deadline_time = deserialized_deadline_alert.reference.evaluate_with(
766-
session=session,
767-
interval=interval,
768-
# TODO : Pretty sure we can drop these last two; verify after testing is complete
769-
dag_id=self.dag_id,
770-
run_id=orm_dagrun.run_id,
761+
team_name = (
762+
DagModel.get_team_name(self.dag_id, session=session)
763+
if airflow_conf.getboolean("core", "multi_team")
764+
else None
765+
)
766+
metrics_tags = prune_dict({"dag_id": self.dag_id, "team_name": team_name})
767+
768+
# Read every value off the ORM up front. A failing alert can leave the session rolled back
769+
# and its instances detached, so nothing after this point may touch ORM state -- otherwise
770+
# the skip path raises DetachedInstanceError and takes the DagRun down anyway.
771+
run_id = orm_dagrun.run_id
772+
dagrun_id = orm_dagrun.id
773+
dagrun_dag_id = orm_dagrun.dag_id
774+
bundle_name = orm_dagrun.dag_model.bundle_name
775+
stored_alerts = [
776+
(alert.id, alert.name, alert.reference, alert.interval, alert.callback_def)
777+
for alert in deadline_alert_records
778+
if alert
779+
]
780+
781+
for alert_id, alert_name, alert_reference, alert_interval, alert_callback_def in stored_alerts:
782+
# Deadline creation is best-effort. Catch per alert so one bad alert cannot starve the rest or abort the DagRun.
783+
try:
784+
deserialized_deadline_alert = decode_deadline_alert(
785+
{
786+
DeadlineAlertFields.NAME: alert_name,
787+
DeadlineAlertFields.REFERENCE: alert_reference,
788+
DeadlineAlertFields.INTERVAL: alert_interval,
789+
DeadlineAlertFields.CALLBACK: alert_callback_def,
790+
}
771791
)
792+
interval = resolve_deadline_alert_interval(deserialized_deadline_alert, session=session)
772793

773-
if deadline_time is not None:
774-
session.add(
775-
Deadline(
776-
deadline_time=deadline_time,
777-
callback=deserialized_deadline_alert.callback,
778-
dagrun_id=orm_dagrun.id,
779-
deadline_alert_id=deadline_alert.id,
780-
dag_id=orm_dagrun.dag_id,
781-
bundle_name=orm_dagrun.dag_model.bundle_name,
782-
)
783-
)
784-
team_name = (
785-
DagModel.get_team_name(self.dag_id, session=session)
786-
if airflow_conf.getboolean("core", "multi_team")
787-
else None
788-
)
789-
stats.incr(
790-
"deadline_alerts.deadline_created",
791-
tags=prune_dict({"dag_id": self.dag_id, "team_name": team_name}),
792-
)
793-
elif required_dagrun_column := _DAGRUN_REFERENCE_REQUIRED_COLUMNS.get(
794-
type(deserialized_deadline_alert.reference)
795-
):
796-
log.warning(
797-
"skipping deadline alert because the deadline reference evaluated to None",
794+
if isinstance(deserialized_deadline_alert.reference, SerializedReferenceModels.TYPES.DAGRUN):
795+
deadline_time = deserialized_deadline_alert.reference.evaluate_with(
796+
session=session,
797+
interval=interval,
798+
# TODO : Pretty sure we can drop these last two; verify after testing is complete
798799
dag_id=self.dag_id,
799-
run_id=orm_dagrun.run_id,
800-
deadline_alert_id=deadline_alert.id,
801-
reference_type=deserialized_deadline_alert.reference.reference_name,
802-
required_dagrun_column=required_dagrun_column,
800+
run_id=run_id,
803801
)
804802

803+
if deadline_time is not None:
804+
session.add(
805+
Deadline(
806+
deadline_time=deadline_time,
807+
callback=deserialized_deadline_alert.callback,
808+
dagrun_id=dagrun_id,
809+
deadline_alert_id=alert_id,
810+
dag_id=dagrun_dag_id,
811+
bundle_name=bundle_name,
812+
)
813+
)
814+
stats.incr("deadline_alerts.deadline_created", tags=metrics_tags)
815+
elif required_dagrun_column := _DAGRUN_REFERENCE_REQUIRED_COLUMNS.get(
816+
type(deserialized_deadline_alert.reference)
817+
):
818+
log.warning(
819+
"skipping deadline alert because the deadline reference evaluated to None",
820+
dag_id=self.dag_id,
821+
run_id=run_id,
822+
deadline_alert_id=alert_id,
823+
reference_type=deserialized_deadline_alert.reference.reference_name,
824+
required_dagrun_column=required_dagrun_column,
825+
)
826+
except Exception as exc:
827+
if isinstance(exc, RuntimeError) and PROHIBIT_COMMIT_ERROR_MESSAGE in str(exc):
828+
# A rejected commit means the scheduler's HA locking is already broken. Recording
829+
# that as a skipped alert would hide it, so let it escape instead.
830+
raise
831+
log.exception(
832+
"skipping deadline alert because creating its deadline failed",
833+
dag_id=self.dag_id,
834+
run_id=run_id,
835+
deadline_alert_id=alert_id,
836+
)
837+
stats.incr("deadline_alerts.deadline_creation_failed", tags=metrics_tags)
838+
805839
@provide_session
806840
def set_task_instance_state(
807841
self,

‎airflow-core/src/airflow/serialization/definitions/deadline.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -406,7 +406,14 @@ def resolve(self, *, session: Session | None = None) -> timedelta:
406406
f"VariableInterval '{self.key}' must be an integer (seconds), got: {value!r}"
407407
) from e
408408

409-
return timedelta(seconds=seconds)
409+
try:
410+
return timedelta(seconds=seconds)
411+
except OverflowError as e:
412+
# ``timedelta`` raises OverflowError, which is not a ValueError, so callers
413+
# inspecting for a bad interval would otherwise miss it.
414+
raise ValueError(
415+
f"VariableInterval '{self.key}' is too large to be a valid interval: {seconds} seconds"
416+
) from e
410417

411418

412419
@attrs.define

‎airflow-core/src/airflow/utils/sqlalchemy.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -682,6 +682,9 @@ def with_db_lock_timeout(session: Session, lock_timeout: int = 30) -> Generator[
682682
session.execute(text(f"SET SESSION innodb_lock_wait_timeout = {old_mysql_timeout}"))
683683

684684

685+
PROHIBIT_COMMIT_ERROR_MESSAGE = "UNEXPECTED COMMIT - THIS WILL BREAK HA LOCKS!"
686+
687+
685688
class CommitProhibitorGuard:
686689
"""Context manager class that powers prohibit_commit."""
687690

@@ -694,7 +697,7 @@ def _validate_commit(self, _):
694697
if self.expected_commit:
695698
self.expected_commit = False
696699
return
697-
raise RuntimeError("UNEXPECTED COMMIT - THIS WILL BREAK HA LOCKS!")
700+
raise RuntimeError(PROHIBIT_COMMIT_ERROR_MESSAGE)
698701

699702
def __enter__(self) -> Self:
700703
event.listen(self.session, "before_commit", self._validate_commit)

‎airflow-core/tests/unit/models/test_dagrun.py‎

Lines changed: 139 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
from collections import defaultdict
2222
from collections.abc import Mapping
2323
from concurrent.futures import ThreadPoolExecutor
24-
from contextlib import contextmanager
24+
from contextlib import contextmanager, nullcontext
2525
from functools import partial, reduce
2626
from typing import TYPE_CHECKING
2727
from unittest import mock
@@ -1954,30 +1954,150 @@ def test_dagrun_deadline_does_not_warn_for_average_runtime_without_history(
19541954
"log_level": "warning",
19551955
} not in caplog
19561956

1957+
@pytest.mark.parametrize(
1958+
("interval", "failure"),
1959+
[
1960+
pytest.param(
1961+
VariableInterval("missing_key"),
1962+
mock.patch.object(Variable, "get", side_effect=KeyError),
1963+
id="unresolvable_interval",
1964+
),
1965+
pytest.param(
1966+
datetime.timedelta(hours=1),
1967+
mock.patch(
1968+
"airflow.serialization.definitions.dag.decode_deadline_alert",
1969+
autospec=True,
1970+
side_effect=ValueError("corrupt deadline alert blob"),
1971+
),
1972+
id="undecodable_alert",
1973+
),
1974+
pytest.param(
1975+
datetime.timedelta(hours=1),
1976+
mock.patch.object(
1977+
SerializedReferenceModels.FixedDatetimeDeadline,
1978+
"evaluate_with",
1979+
autospec=True,
1980+
side_effect=RuntimeError("evaluate_with failed"),
1981+
),
1982+
id="unevaluable_reference",
1983+
),
1984+
],
1985+
)
1986+
@mock.patch("airflow._shared.observability.metrics.stats.incr")
19571987
@mock.patch.object(Deadline, "prune_deadlines")
1958-
def test_dagrun_deadline_variable_interval_missing_variable_fails(self, _, session, deadline_test_dag):
1988+
def test_dagrun_deadline_failure_does_not_abort_dagrun(
1989+
self, _, mock_stats_incr, interval, failure, session, deadline_test_dag
1990+
):
1991+
future_date = datetime.datetime(2037, 1, 1, tzinfo=datetime.UTC)
19591992

1960-
with mock.patch.object(
1961-
Variable,
1962-
"get",
1963-
side_effect=KeyError,
1993+
scheduler_dag = deadline_test_dag(
1994+
deadline=DeadlineAlert(
1995+
reference=DeadlineReference.FIXED_DATETIME(future_date),
1996+
interval=interval,
1997+
callback=AsyncCallback(empty_callback_for_deadline),
1998+
),
1999+
)
2000+
2001+
with (
2002+
conf_vars({("core", "multi_team"): "true"}),
2003+
mock.patch("airflow.models.dag.DagModel.get_team_name", return_value="team_alpha"),
2004+
failure,
19642005
):
1965-
future_date = datetime.datetime.now() + datetime.timedelta(days=365)
2006+
dag_run = self.create_dag_run(
2007+
dag=scheduler_dag,
2008+
task_states={"task_1": TaskInstanceState.SUCCESS},
2009+
session=session,
2010+
)
2011+
2012+
assert dag_run is not None
2013+
assert session.execute(select(Deadline)).scalars().one_or_none() is None
2014+
mock_stats_incr.assert_any_call(
2015+
"deadline_alerts.deadline_creation_failed",
2016+
tags={"dag_id": scheduler_dag.dag_id, "team_name": "team_alpha"},
2017+
)
19662018

1967-
scheduler_dag = deadline_test_dag(
1968-
deadline=DeadlineAlert(
1969-
reference=DeadlineReference.FIXED_DATETIME(future_date),
1970-
interval=VariableInterval("missing_key"),
1971-
callback=AsyncCallback(empty_callback_for_deadline),
2019+
@mock.patch("airflow._shared.observability.metrics.stats.incr")
2020+
@mock.patch.object(Deadline, "prune_deadlines")
2021+
@mock.patch.object(Variable, "get")
2022+
def test_dagrun_deadline_failure_that_detaches_orm_objects_still_skips(
2023+
self, mock_variable_get, _, mock_stats_incr, session, deadline_test_dag
2024+
):
2025+
"""A failing interval resolution can roll back and close the caller's session underneath
2026+
the handler (``create_session`` reusing the scoped session does exactly that), detaching
2027+
every ORM instance. The skip path must still log and count without touching ORM state."""
2028+
scheduler_dag = deadline_test_dag(
2029+
deadline=DeadlineAlert(
2030+
reference=DeadlineReference.FIXED_DATETIME(
2031+
datetime.datetime(2037, 1, 1, tzinfo=datetime.UTC)
19722032
),
1973-
)
2033+
interval=VariableInterval("missing_key"),
2034+
callback=AsyncCallback(empty_callback_for_deadline),
2035+
),
2036+
)
2037+
# The alert rows must be persistent from an earlier transaction (as in the live
2038+
# scheduler, where the dag processor committed them) so the rollback below expires
2039+
# them and the close detaches them instead of returning them to transient.
2040+
session.commit()
19742041

1975-
with pytest.raises(ValueError, match="not found"):
1976-
self.create_dag_run(
1977-
dag=scheduler_dag,
1978-
task_states={"task_1": TaskInstanceState.SUCCESS},
1979-
session=session,
1980-
)
2042+
def teardown_session_and_raise(*args, **kwargs):
2043+
session.rollback()
2044+
session.close()
2045+
raise KeyError("missing_key")
2046+
2047+
mock_variable_get.side_effect = teardown_session_and_raise
2048+
2049+
dag_run = self.create_dag_run(dag=scheduler_dag, session=session)
2050+
2051+
assert dag_run is not None
2052+
mock_stats_incr.assert_any_call(
2053+
"deadline_alerts.deadline_creation_failed",
2054+
tags={"dag_id": scheduler_dag.dag_id},
2055+
)
2056+
2057+
@pytest.mark.parametrize(
2058+
("interval", "expect_deadline", "variable_value"),
2059+
[
2060+
pytest.param(datetime.timedelta(seconds=5), True, None, id="deadline_created"),
2061+
pytest.param(VariableInterval("missing_key"), False, None, id="deadline_skipped"),
2062+
pytest.param(VariableInterval("resolvable_key"), True, "4200", id="variable_interval_resolved"),
2063+
],
2064+
)
2065+
@mock.patch.object(Deadline, "prune_deadlines")
2066+
def test_dagrun_deadline_handling_does_not_commit(
2067+
self, _, interval, expect_deadline, variable_value, session, deadline_test_dag
2068+
):
2069+
"""DagRuns are created under ``prohibit_commit``, so creating a deadline, skipping one whose
2070+
interval will not resolve, and resolving a ``VariableInterval`` out of the metadata database
2071+
all have to leave the scheduler's session uncommitted."""
2072+
future_date = datetime.datetime(2037, 1, 1, tzinfo=datetime.UTC)
2073+
scheduler_dag = deadline_test_dag(
2074+
deadline=DeadlineAlert(
2075+
reference=DeadlineReference.FIXED_DATETIME(future_date),
2076+
interval=interval,
2077+
callback=AsyncCallback(empty_callback_for_deadline),
2078+
),
2079+
)
2080+
2081+
if variable_value is None:
2082+
variable_lookup = mock.patch.object(Variable, "get", side_effect=KeyError)
2083+
else:
2084+
# A real row committed before the guard opens, so resolve() has to go through
2085+
# MetastoreBackend. Drop the session forwarding and the backend opens the same scoped
2086+
# session and commits it, which the guard rejects and the lookup then reports as missing.
2087+
Variable.set(key=interval.key, value=variable_value, session=session)
2088+
session.commit()
2089+
variable_lookup = nullcontext()
2090+
2091+
# No task_states: the helper's get_task_instance() opens its own session and would trip the guard.
2092+
with prohibit_commit(session), variable_lookup:
2093+
dag_run = self.create_dag_run(dag=scheduler_dag, session=session)
2094+
session.flush()
2095+
2096+
assert dag_run is not None
2097+
deadline = session.execute(select(Deadline)).scalars().one_or_none()
2098+
assert (deadline is not None) is expect_deadline
2099+
if variable_value is not None:
2100+
assert deadline.deadline_time == future_date + datetime.timedelta(seconds=int(variable_value))
19812101

19822102

19832103
@pytest.mark.parametrize(

‎airflow-core/tests/unit/serialization/definitions/test_deadline.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ def test_resolve_forwards_session(self, mocker):
6161
(None, True, "not found"),
6262
("abc", False, "must be an integer"),
6363
("", False, "must be an integer"),
64+
("99999999999999", False, "too large to be a valid interval"),
6465
],
6566
)
6667
def test_resolve_invalid(self, mocker, value, raise_missing, match):

‎shared/observability/src/airflow_shared/observability/metrics/metrics_template.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,12 @@ metrics:
326326
legacy_name: "-"
327327
name_variables: []
328328

329+
- name: "deadline_alerts.deadline_creation_failed"
330+
description: "Number of deadline alerts skipped because their deadline could not be created"
331+
type: "counter"
332+
legacy_name: "-"
333+
name_variables: []
334+
329335
- name: "deadline_alerts.deadline_missed"
330336
description: "Number of deadline alerts that fired because a Dag run missed its deadline"
331337
type: "counter"

0 commit comments

Comments
 (0)