Skip to content

Commit 637b741

Browse files
Pass Dag runs directly to Deadline references (#70714)
* Fix custom Deadline references with optional keyword arguments Preserve the Dag run evaluation context for custom references that accept additional keyword arguments, so they continue to evaluate correctly. Co-authored-by: D. Ferruzzi <[email protected]>
1 parent 61f1d4b commit 637b741

9 files changed

Lines changed: 224 additions & 436 deletions

File tree

‎airflow-core/docs/howto/deadline-alerts.rst‎

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -466,24 +466,19 @@ Place the reference classes and the plugin that registers them in your plugins f
466466
class MyCustomDecoratedReference(BaseDeadlineReference):
467467
"""A custom reference evaluated when Dag runs are created."""
468468
469-
def _evaluate_with(self, *, session: Session, **kwargs) -> datetime:
470-
# Add your business logic here
471-
return your_datetime
469+
def _evaluate_with(self, *, session: Session, dagrun) -> datetime:
470+
my_datetime = my_business_logic(dagrun.logical_date)
471+
return my_datetime
472472
473473
474474
# You can specify when evaluate_with will be called by providing a DeadlineReference.TYPES value.
475475
@deadline_reference(DeadlineReference.TYPES.DAGRUN_QUEUED)
476476
class MyQueuedReference(BaseDeadlineReference):
477477
"""A custom reference evaluated when Dag runs are queued."""
478478
479-
# Ask for the Dag run context values supplied by Airflow; see notes below.
480-
required_kwargs = {"dag_id", "run_id"}
481-
482-
def _evaluate_with(self, *, session: Session, **kwargs) -> datetime:
483-
dag_id = kwargs["dag_id"]
484-
run_id = kwargs["run_id"]
485-
# Use dag_id and run_id in your calculation
486-
return your_datetime
479+
def _evaluate_with(self, *, session: Session, dagrun) -> datetime:
480+
my_datetime = my_business_logic(dagrun.queued_at)
481+
return my_datetime
487482
488483
489484
# Register the classes so the scheduler can resolve them when it deserializes the Dag.
@@ -564,8 +559,8 @@ followed by a more urgent escalation if the Dag is still running.
564559
* **Plugin Registration**: Custom references must be listed in the ``deadline_references`` attribute
565560
of an ``AirflowPlugin``, so the plugins directory is the natural home for them.
566561
* **API Server Restart**: Restart the Airflow API Server after adding or modifying custom references.
567-
* **Required Parameters**: ``required_kwargs`` declares which Dag run context values Airflow should
568-
forward to ``_evaluate_with()``. Only ``dag_id`` and ``run_id`` are available; declaring anything
569-
else raises a ``ValueError`` when the deadline is evaluated. To configure a reference itself, give
570-
it constructor fields or read from an Airflow Variable.
562+
* **Dag run context**: Add a ``dagrun`` parameter to ``_evaluate_with()`` to use the Dag run being evaluated.
563+
* **Required Parameters** *(deprecated)*: ``required_kwargs`` declares which parameters your reference
564+
needs. It still works but is deprecated and will be removed in Airflow 4.0 — declare the keyword-only
565+
parameters your ``_evaluate_with()`` needs instead.
571566
* **Database Access**: Use the ``session`` parameter for Airflow database queries if needed.
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Deprecate the ``required_kwargs`` attribute for custom deadline references. Implement ``_evaluate_with()`` with a ``dagrun`` parameter instead.

‎airflow-core/src/airflow/models/deadline.py‎

Lines changed: 48 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -17,21 +17,23 @@
1717
from __future__ import annotations
1818

1919
import logging
20+
import warnings
2021
from abc import ABC, abstractmethod
2122
from collections.abc import Sequence
2223
from dataclasses import dataclass
2324
from datetime import datetime, timedelta
24-
from typing import TYPE_CHECKING, Any, cast
25+
from inspect import signature
26+
from typing import TYPE_CHECKING, Any, Protocol, cast
2527
from uuid import UUID
2628

2729
import uuid6
28-
from sqlalchemy import Boolean, ForeignKey, Index, Integer, Uuid, and_, func, inspect, select, text
29-
from sqlalchemy.exc import SQLAlchemyError
30+
from sqlalchemy import Boolean, ForeignKey, Index, Integer, Uuid, and_, func, select, text
3031
from sqlalchemy.orm import Mapped, mapped_column, relationship
3132

3233
from airflow._shared.observability.metrics import stats
3334
from airflow._shared.timezones import timezone
3435
from airflow.configuration import conf
36+
from airflow.exceptions import RemovedInAirflow4Warning
3537
from airflow.models.base import Base
3638
from airflow.models.callback import (
3739
Callback,
@@ -57,6 +59,38 @@
5759
CALLBACK_METRICS_PREFIX = "deadline_alerts"
5860

5961

62+
class DeadlineDagRunProtocol(Protocol):
63+
"""The Dag run interface deadline references are evaluated against."""
64+
65+
dag_id: str
66+
logical_date: datetime | None
67+
queued_at: datetime | None
68+
69+
70+
def _get_evaluation_kwargs(reference: Any, evaluator: Any, kwargs: dict[str, Any]) -> dict[str, Any]:
71+
"""Return the evaluation arguments accepted by a deadline reference."""
72+
required_kwargs: set[str] | None = getattr(reference, "required_kwargs", None)
73+
if required_kwargs is not None:
74+
warnings.warn(
75+
"required_kwargs is deprecated. Declare the keyword-only parameters your "
76+
"_evaluate_with() implementation needs instead.",
77+
RemovedInAirflow4Warning,
78+
stacklevel=3,
79+
)
80+
kwargs = {key: value for key, value in kwargs.items() if key in required_kwargs}
81+
if missing_kwargs := required_kwargs - kwargs.keys():
82+
raise ValueError(
83+
f"{reference.__class__.__name__} is missing required parameters: {', '.join(missing_kwargs)}"
84+
)
85+
return kwargs
86+
87+
parameters = signature(evaluator).parameters
88+
if any(param.kind is param.VAR_KEYWORD for param in parameters.values()):
89+
return kwargs
90+
91+
return {key: value for key, value in kwargs.items() if key in parameters}
92+
93+
6094
class classproperty:
6195
"""
6296
Decorator that converts a method with a single cls argument into a property.
@@ -323,34 +357,18 @@ def get_reference_class(cls, reference_name: str) -> type[BaseDeadlineReference]
323357
class BaseDeadlineReference(LoggingMixin, ABC):
324358
"""Base class for all Deadline implementations."""
325359

326-
# Set of required kwargs - subclasses should override this.
327-
required_kwargs: set[str] = set()
328-
329360
@classproperty
330361
def reference_name(cls: Any) -> str:
331362
return cls.__name__
332363

333364
def evaluate_with(self, *, session: Session, interval: timedelta, **kwargs: Any) -> datetime | None:
334-
"""Validate the provided kwargs and evaluate this deadline with the given conditions."""
335-
filtered_kwargs = {k: v for k, v in kwargs.items() if k in self.required_kwargs}
336-
337-
if missing_kwargs := self.required_kwargs - filtered_kwargs.keys():
338-
raise ValueError(
339-
f"{self.__class__.__name__} is missing required parameters: {', '.join(missing_kwargs)}"
340-
)
341-
342-
if extra_kwargs := kwargs.keys() - filtered_kwargs.keys():
343-
self.log.debug(
344-
"%s ignoring unexpected parameters: %s",
345-
self.reference_name,
346-
", ".join(extra_kwargs),
347-
)
348-
349-
base_time = self._evaluate_with(session=session, **filtered_kwargs)
365+
"""Evaluate this deadline with the supplied context."""
366+
evaluation_kwargs = _get_evaluation_kwargs(self, self._evaluate_with, kwargs)
367+
base_time = self._evaluate_with(session=session, **evaluation_kwargs)
350368
return base_time + interval if base_time is not None else None
351369

352370
@abstractmethod
353-
def _evaluate_with(self, *, session: Session, **kwargs: Any) -> datetime | None:
371+
def _evaluate_with(self, *, session: Session, dagrun: DeadlineDagRunProtocol) -> datetime | None:
354372
"""Must be implemented by subclasses to perform the actual evaluation."""
355373
raise NotImplementedError
356374

@@ -384,7 +402,7 @@ class FixedDatetimeDeadline(BaseDeadlineReference):
384402

385403
_datetime: datetime
386404

387-
def _evaluate_with(self, *, session: Session, **kwargs: Any) -> datetime | None:
405+
def _evaluate_with(self, *, session: Session, **kwargs) -> datetime | None:
388406
return self._datetime
389407

390408
def serialize_reference(self) -> dict:
@@ -400,23 +418,14 @@ def deserialize_reference(cls, reference_data: dict):
400418
class DagRunLogicalDateDeadline(BaseDeadlineReference):
401419
"""A deadline that returns a DagRun's logical date."""
402420

403-
required_kwargs = {"dag_id", "run_id"}
404-
405-
def _evaluate_with(self, *, session: Session, **kwargs: Any) -> datetime | None:
406-
from airflow.models import DagRun
407-
408-
return _fetch_from_db(DagRun.logical_date, session=session, **kwargs)
421+
def _evaluate_with(self, *, session: Session, dagrun: DeadlineDagRunProtocol) -> datetime | None:
422+
return dagrun.logical_date
409423

410424
class DagRunQueuedAtDeadline(BaseDeadlineReference):
411425
"""A deadline that returns when a DagRun was queued."""
412426

413-
required_kwargs = {"dag_id", "run_id"}
414-
415-
@provide_session
416-
def _evaluate_with(self, *, session: Session, **kwargs: Any) -> datetime | None:
417-
from airflow.models import DagRun
418-
419-
return _fetch_from_db(DagRun.queued_at, session=session, **kwargs)
427+
def _evaluate_with(self, *, session: Session, dagrun: DeadlineDagRunProtocol) -> datetime | None:
428+
return dagrun.queued_at
420429

421430
@dataclass
422431
class AverageRuntimeDeadline(BaseDeadlineReference):
@@ -425,7 +434,6 @@ class AverageRuntimeDeadline(BaseDeadlineReference):
425434
DEFAULT_LIMIT = 10
426435
max_runs: int
427436
min_runs: int | None = None
428-
required_kwargs = {"dag_id"}
429437

430438
def __post_init__(self):
431439
if self.min_runs is None:
@@ -434,10 +442,10 @@ def __post_init__(self):
434442
raise ValueError("min_runs must be at least 1")
435443

436444
@provide_session
437-
def _evaluate_with(self, *, session: Session, **kwargs: Any) -> datetime | None:
445+
def _evaluate_with(self, *, session: Session, dagrun: DeadlineDagRunProtocol) -> datetime | None:
438446
from airflow.models import DagRun
439447

440-
dag_id = kwargs["dag_id"]
448+
dag_id = dagrun.dag_id
441449

442450
# Get database dialect to use appropriate time difference calculation
443451
dialect = get_dialect_name(session)
@@ -510,54 +518,3 @@ def deserialize_reference(cls, reference_data: dict):
510518

511519

512520
DeadlineReferenceType = ReferenceModels.BaseDeadlineReference
513-
514-
515-
@provide_session
516-
def _fetch_from_db(model_reference: Mapped, *, session=None, **conditions) -> datetime | None:
517-
"""
518-
Fetch a datetime value from the database using the provided model reference and filtering conditions.
519-
520-
For example, to fetch a TaskInstance's start_date:
521-
_fetch_from_db(
522-
TaskInstance.start_date, dag_id='example_dag', task_id='example_task', run_id='example_run'
523-
)
524-
525-
This generates SQL equivalent to:
526-
SELECT start_date
527-
FROM task_instance
528-
WHERE dag_id = 'example_dag'
529-
AND task_id = 'example_task'
530-
AND run_id = 'example_run'
531-
532-
:param model_reference: SQLAlchemy Column to select (e.g., DagRun.logical_date, TaskInstance.start_date)
533-
:param conditions: Filtering conditions applied as equality comparisons in the WHERE clause.
534-
Multiple conditions are combined with AND.
535-
:param session: SQLAlchemy session (auto-provided by decorator)
536-
"""
537-
query = select(model_reference)
538-
539-
for key, value in conditions.items():
540-
inspected = inspect(model_reference)
541-
if inspected is not None:
542-
query = query.where(getattr(inspected.class_, key) == value)
543-
544-
compiled_query = query.compile(compile_kwargs={"literal_binds": True})
545-
pretty_query = "\n ".join(str(compiled_query).splitlines())
546-
logger.debug(
547-
"Executing query:\n %r\nAs SQL:\n %s",
548-
query,
549-
pretty_query,
550-
)
551-
552-
try:
553-
result = session.scalar(query)
554-
except SQLAlchemyError:
555-
logger.exception("Database query failed.")
556-
raise
557-
558-
if result is None:
559-
message = f"No matching record found in the database for query:\n {pretty_query}"
560-
logger.error(message)
561-
raise ValueError(message)
562-
563-
return result

‎airflow-core/src/airflow/models/taskinstance.py‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -255,14 +255,11 @@ def _add_and_prime_mapped_ti(
255255
set_committed_value(ti, "dag_run", dag_run)
256256

257257

258-
def _recalculate_dagrun_queued_at_deadlines(
259-
dagrun: DagRun, new_queued_at: datetime, session: Session
260-
) -> None:
258+
def _recalculate_dagrun_queued_at_deadlines(dagrun: DagRun, *, session: Session) -> None:
261259
"""
262260
Recalculate deadline times for deadlines that reference dagrun.queued_at.
263261
264262
:param dagrun: The DagRun whose deadlines should be recalculated
265-
:param new_queued_at: The new queued_at timestamp to use for calculation
266263
:param session: Database session
267264
268265
:meta private:
@@ -285,12 +282,9 @@ def _recalculate_dagrun_queued_at_deadlines(
285282
from airflow.serialization.decoders import decode_deadline_alert_model, resolve_deadline_alert_interval
286283

287284
for deadline, deadline_alert in results:
288-
# We can't use evaluate_with() since the new queued_at is not written to the DB yet, and
289-
# interval is stored as JSON, so it has to be decoded rather than passed to timedelta().
290285
try:
291-
interval = resolve_deadline_alert_interval(
292-
decode_deadline_alert_model(deadline_alert), session=session
293-
)
286+
decoded_alert = decode_deadline_alert_model(deadline_alert)
287+
interval = resolve_deadline_alert_interval(decoded_alert, session=session)
294288
except (ValueError, TypeError):
295289
# Either step can fail: the alert may hold a payload the decoder refuses, or a
296290
# variable-backed interval may point at a Variable that is missing or non-numeric.
@@ -304,7 +298,16 @@ def _recalculate_dagrun_queued_at_deadlines(
304298
)
305299
continue
306300

307-
new_deadline_time = new_queued_at + interval
301+
new_deadline_time = decoded_alert.reference.evaluate_with(
302+
session=session,
303+
interval=interval,
304+
dagrun=dagrun,
305+
dag_id=dagrun.dag_id,
306+
run_id=dagrun.run_id,
307+
)
308+
309+
if new_deadline_time is None:
310+
continue
308311

309312
log.debug(
310313
"Recalculating deadline %s for DagRun %s.%s: old=%s, new=%s",
@@ -603,7 +606,7 @@ def get_cached_latest_dag_version(dag_id: str) -> DagVersion | None:
603606
parent_context=parent_trace_context(dr.conf),
604607
)
605608

606-
_recalculate_dagrun_queued_at_deadlines(dr, dr.queued_at, session)
609+
_recalculate_dagrun_queued_at_deadlines(dr, session=session)
607610

608611
if was_finished:
609612
dr.state = dag_run_state

‎airflow-core/src/airflow/serialization/definitions/dag.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -795,7 +795,7 @@ def _process_dagrun_deadline_alerts(
795795
deadline_time = deserialized_deadline_alert.reference.evaluate_with(
796796
session=session,
797797
interval=interval,
798-
# TODO : Pretty sure we can drop these last two; verify after testing is complete
798+
dagrun=orm_dagrun,
799799
dag_id=self.dag_id,
800800
run_id=run_id,
801801
)

0 commit comments

Comments
 (0)