Skip to content

Commit 8a5212c

Browse files
authored
Fix Dag run filtering for null start dates (#71654)
* Fix Dag run filtering for null start dates * Cover Dag list start date filtering with null latest runs * Preserve queued Dag filtering semantics * Add unit test for nullable datetime lower bound clause * Cover nullable datetime strict lower bounds * Use UTC in nullable datetime filter tests
1 parent 3199497 commit 8a5212c

4 files changed

Lines changed: 74 additions & 10 deletions

File tree

‎airflow-core/src/airflow/api_fastapi/common/parameters/range.py‎

Lines changed: 30 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939

4040
if TYPE_CHECKING:
4141
from sqlalchemy.orm.attributes import InstrumentedAttribute
42-
from sqlalchemy.sql import Select
42+
from sqlalchemy.sql import ColumnElement, Select
4343

4444

4545
def _safe_parse_datetime(date_to_check: str) -> datetime:
@@ -136,12 +136,27 @@ class NullableDatetimeRangeFilter(RangeFilter):
136136
``OR`` predicates so each branch can be satisfied by an independent index scan.
137137
138138
NULL semantics: ``start_date=NULL`` means the task has not started yet; ``end_date=NULL`` means
139-
the task is still running. For lower bounds the NULL branch passes unconditionally — a not-yet-
140-
started/ended task will eventually satisfy any past lower bound. For upper bounds the NULL branch
141-
is ``col IS NULL AND now() <= x``, preserving the COALESCE(col, now()) semantics without the
142-
function-wrap index penalty.
139+
the task is still running. For lower bounds the NULL branch usually passes unconditionally, but
140+
callers may narrow it with an extra clause when NULL cannot still evolve into an in-range value.
141+
For upper bounds the NULL branch is ``col IS NULL AND now() <= x``, preserving the
142+
COALESCE(col, now()) semantics without the function-wrap index penalty.
143143
"""
144144

145+
def __init__(
146+
self,
147+
value: Range | None,
148+
attribute: InstrumentedAttribute,
149+
null_lower_bound_clause: ColumnElement[bool] | None = None,
150+
) -> None:
151+
super().__init__(value, attribute)
152+
self.null_lower_bound_clause = null_lower_bound_clause
153+
154+
def _get_null_lower_bound_clause(self) -> ColumnElement[bool]:
155+
null_clause: ColumnElement[bool] = self.attribute.is_(None)
156+
if self.null_lower_bound_clause is not None:
157+
null_clause = and_(null_clause, self.null_lower_bound_clause)
158+
return null_clause
159+
145160
def to_orm(self, select: Select) -> Select:
146161
if self.skip_none is False:
147162
raise ValueError(f"Cannot set 'skip_none' to False on a {type(self)}")
@@ -151,10 +166,10 @@ def to_orm(self, select: Select) -> Select:
151166

152167
if self.value.lower_bound_gte:
153168
x = self.value.lower_bound_gte
154-
select = select.where(or_(self.attribute >= x, self.attribute.is_(None)))
169+
select = select.where(or_(self.attribute >= x, self._get_null_lower_bound_clause()))
155170
if self.value.lower_bound_gt:
156171
x = self.value.lower_bound_gt
157-
select = select.where(or_(self.attribute > x, self.attribute.is_(None)))
172+
select = select.where(or_(self.attribute > x, self._get_null_lower_bound_clause()))
158173
if self.value.upper_bound_lte:
159174
x = self.value.upper_bound_lte
160175
select = select.where(or_(self.attribute <= x, and_(self.attribute.is_(None), func.now() <= x)))
@@ -166,7 +181,11 @@ def to_orm(self, select: Select) -> Select:
166181

167182

168183
def datetime_range_filter_factory(
169-
filter_name: str, model: Base, attribute_name: str | None = None
184+
filter_name: str,
185+
model: Base,
186+
attribute_name: str | None = None,
187+
*,
188+
null_lower_bound_clause: ColumnElement[bool] | None = None,
170189
) -> Callable[[datetime | None, datetime | None, datetime | None, datetime | None], RangeFilter]:
171190
def depends_datetime(
172191
lower_bound_gte: datetime | None = Query(alias=f"{filter_name}_gte", default=None),
@@ -182,7 +201,9 @@ def depends_datetime(
182201
upper_bound_lt=upper_bound_lt,
183202
)
184203
if filter_name in ("start_date", "end_date"):
185-
return NullableDatetimeRangeFilter(range_val, attr)
204+
return NullableDatetimeRangeFilter(
205+
range_val, attr, null_lower_bound_clause=null_lower_bound_clause
206+
)
186207
return RangeFilter(range_val, attr)
187208

188209
return depends_datetime

‎airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_run.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -518,7 +518,14 @@ def get_dag_runs(
518518
offset: QueryOffset,
519519
run_after: Annotated[RangeFilter, Depends(datetime_range_filter_factory("run_after", DagRun))],
520520
logical_date: Annotated[RangeFilter, Depends(datetime_range_filter_factory("logical_date", DagRun))],
521-
start_date_range: Annotated[RangeFilter, Depends(datetime_range_filter_factory("start_date", DagRun))],
521+
start_date_range: Annotated[
522+
RangeFilter,
523+
Depends(
524+
datetime_range_filter_factory(
525+
"start_date", DagRun, null_lower_bound_clause=DagRun.end_date.is_(None)
526+
)
527+
),
528+
],
522529
end_date_range: Annotated[RangeFilter, Depends(datetime_range_filter_factory("end_date", DagRun))],
523530
duration_range: Annotated[RangeFilter, Depends(float_range_filter_factory("duration", DagRun))],
524531
update_at_range: Annotated[RangeFilter, Depends(datetime_range_filter_factory("updated_at", DagRun))],

‎airflow-core/tests/unit/api_fastapi/common/test_parameters.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -630,6 +630,25 @@ def test_lower_bound_does_not_include_now(self):
630630
assert "now()" not in sql
631631
assert "coalesce" not in sql
632632

633+
@pytest.mark.parametrize("lower_bound_name", ["lower_bound_gte", "lower_bound_gt"])
634+
def test_null_lower_bound_clause_is_added_to_null_branch(self, lower_bound_name):
635+
"""Custom NULL lower-bound clause is ANDed with the attribute IS NULL check."""
636+
bound = datetime(2026, 5, 3, 12, 0, 0, tzinfo=UTC)
637+
depends = datetime_range_filter_factory(
638+
"start_date",
639+
DagRun,
640+
null_lower_bound_clause=DagRun.end_date.is_(None),
641+
)
642+
bounds = dict(lower_bound_gte=None, lower_bound_gt=None, upper_bound_lte=None, upper_bound_lt=None)
643+
bounds[lower_bound_name] = bound
644+
rf = depends(**bounds)
645+
646+
sql = _compile(rf.to_orm(select(DagRun)))
647+
648+
assert "start_date is null" in sql
649+
assert "end_date is null" in sql
650+
assert "start_date is null and" in sql
651+
633652
def test_upper_bound_includes_now_for_running_tasks(self):
634653
"""NULL branch on upper bounds uses now() to proxy the in-progress task's current time."""
635654
bound = datetime(2026, 5, 3, 12, 0, 0, tzinfo=UTC)

‎airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -834,6 +834,23 @@ def test_cursor_pagination_nullable_sort_column_returns_all_rows(self, test_clie
834834
assert len(collected) == len(set(collected)), "cursor pages overlapped"
835835
assert set(collected) == full_ids, "cursor pagination dropped rows across the NULL boundary"
836836

837+
@pytest.mark.usefixtures("configure_git_connection_for_dag_bundle")
838+
def test_start_date_lower_bound_excludes_finished_runs_with_null_start_date(self, test_client, session):
839+
run = session.scalar(select(DagRun).where(DagRun.run_id == DAG1_RUN2_ID))
840+
run.start_date = None
841+
run.end_date = START_DATE1 + timedelta(minutes=5)
842+
session.commit()
843+
844+
response = test_client.get(
845+
"/dags/~/dagRuns",
846+
params={"start_date_gte": START_DATE2.isoformat(), "order_by": "dag_run_id"},
847+
)
848+
849+
assert response.status_code == 200, response.json()
850+
body = response.json()
851+
assert body["total_entries"] == 2
852+
assert [run["dag_run_id"] for run in body["dag_runs"]] == [DAG2_RUN1_ID, DAG2_RUN2_ID]
853+
837854
@pytest.mark.parametrize(
838855
("dag_id", "query_params", "expected_dag_id_list"),
839856
[

0 commit comments

Comments
 (0)