Skip to content

Commit 675b02a

Browse files
authored
Make airflow-core typing hold for SQLAlchemy 2.0 and 2.1 (#74629)
SQLAlchemy 2.1 turns Select/Row/Result into TypeVarTuple generics, so `Select[tuple[X]]` now means a one-column select of a tuple, and rows unpack to the real (possibly nullable) column types instead of Any. No parameterised `Select` spelling is valid on both versions, so helpers that only refine a select they receive are generic over it, and helpers that build their own select return an unparameterised `Select`. Nullable columns are narrowed with `is None` checks rather than cast; the pool and asset-event queries already exclude NULL there, while get_authorized_pools now skips pools whose name is NULL. The 12-column task instance row, beyond SQLAlchemy's typed select() overloads, is unpacked into locals declared with their column types. Type-checks with mypy on both SQLAlchemy 2.0.54 and 2.1.4.
1 parent 7bf4936 commit 675b02a

11 files changed

Lines changed: 73 additions & 23 deletions

File tree

‎airflow-core/src/airflow/api_fastapi/auth/managers/base_auth_manager.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -786,7 +786,8 @@ def get_authorized_pools(
786786
rows = session.execute(stmt).all()
787787
pools_by_team: dict[str | None, set[str]] = defaultdict(set)
788788
for pool_name, team_name in rows:
789-
pools_by_team[team_name].add(pool_name)
789+
if pool_name is not None:
790+
pools_by_team[team_name].add(pool_name)
790791

791792
pool_names: set[str] = set()
792793
for team_name, team_pool_names in pools_by_team.items():

‎airflow-core/src/airflow/api_fastapi/core_api/services/ui/dependencies.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -417,7 +417,9 @@ def get_data_dependencies(
417417
select(TaskInletAssetReference.dag_id).where(TaskInletAssetReference.asset_id == asset_id),
418418
select(DagScheduleAssetReference.dag_id).where(DagScheduleAssetReference.asset_id == asset_id),
419419
)
420-
connected_dag_ids = set(session.scalars(select(connected_dag_ids_query.subquery().c.dag_id)))
420+
connected_dag_ids: set[str] = set(
421+
session.scalars(select(connected_dag_ids_query.subquery().c.dag_id))
422+
)
421423
if not connected_dag_ids & readable_dag_ids:
422424
return {"nodes": [], "edges": []}
423425

@@ -631,6 +633,8 @@ def get_data_dependencies(
631633
.distinct()
632634
)
633635
for event_asset_id, source_dag_id, source_task_id in alias_events:
636+
if source_dag_id is None or source_task_id is None:
637+
continue
634638
if readable_dag_ids is not None and source_dag_id not in readable_dag_ids:
635639
continue
636640
asset_node_id = f"asset:{event_asset_id}"

‎airflow-core/src/airflow/api_fastapi/execution_api/routes/task_instances.py‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import json
2525
from collections import defaultdict
2626
from collections.abc import Callable, Collection, Iterator, Sequence
27+
from datetime import datetime
2728
from typing import Annotated, Any, NoReturn, cast
2829
from uuid import UUID
2930

@@ -347,7 +348,7 @@ def ti_run(
347348
# backend.
348349

349350
# However, do not clear it for deferral
350-
xcom_keys = []
351+
xcom_keys: list[str] = []
351352
if not ti.next_method:
352353
read = build_xcom_read_query(producer_ids=select(TI.id).where(TI.id == task_instance_id))
353354
entity = xcom_entity(read)
@@ -569,6 +570,19 @@ def ti_update_state(
569570
.with_for_update(of=TI)
570571
.execution_options(include_all_attempts=True)
571572
)
573+
# A 12-column select exceeds SQLAlchemy's typed ``select()`` overloads, so the row is untyped.
574+
previous_state: str | None
575+
try_number: int
576+
max_tries: int
577+
dag_id: str
578+
task_id: str
579+
run_id: str
580+
region_index: int
581+
region_id: UUID
582+
hostname: str | None
583+
logical_date: datetime | None
584+
owners: str | None
585+
working_set: bool | None
572586
try:
573587
(
574588
previous_state,
@@ -583,7 +597,7 @@ def ti_update_state(
583597
logical_date,
584598
owners,
585599
working_set,
586-
) = session.execute(old).one()
600+
) = tuple(session.execute(old).one())
587601
log.debug(
588602
"Retrieved current task instance state",
589603
previous_state=previous_state,
@@ -1122,7 +1136,7 @@ def ti_skip_downstream(
11221136
raise HTTPException(status.HTTP_400_BAD_REQUEST, str(error)) from error
11231137
if not selects:
11241138
return
1125-
selected_ids = set(session.scalars(union(*selects)))
1139+
selected_ids: set[UUID] = set(session.scalars(union(*selects)))
11261140

11271141
# Don't overwrite tasks that are already executing or finished.
11281142
# See: https://github.com/apache/airflow/issues/59378

‎airflow-core/src/airflow/jobs/scheduler_job_runner.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3588,7 +3588,7 @@ def _get_num_times_stuck_in_queued(self, ti: TaskInstance, *, session: Session =
35883588
count_result: int | None = session.scalar(statement)
35893589
return count_result if count_result is not None else 0
35903590

3591-
previous_ti_metrics: dict[TaskInstanceState, dict[tuple[str, str, str], int]] = {}
3591+
previous_ti_metrics: dict[TaskInstanceState, dict[tuple[str, str, str | None], int]] = {}
35923592

35933593
@provide_session
35943594
def _emit_ti_metrics(self, *, session: Session = NEW_SESSION) -> None:

‎airflow-core/src/airflow/models/dynamic_region.py‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
from collections.abc import Collection, Iterable, Iterator
2222
from datetime import datetime
2323
from functools import partial
24-
from typing import TYPE_CHECKING, Any
24+
from typing import TYPE_CHECKING, TypeVar
2525
from uuid import UUID
2626

2727
import attrs
@@ -72,6 +72,11 @@
7272
from sqlalchemy.sql.elements import ColumnElement
7373

7474
from airflow.models.taskinstance import TaskInstance
75+
from airflow.utils.sqlalchemy import UUIDSelect
76+
77+
# Bound unparameterised: no ``Select[...]`` spelling accepts every arity on both SQLAlchemy 2.0 and 2.1.
78+
# TODO: parameterise the bound with the variadic Select once the minimum SQLAlchemy is 2.1.
79+
_SelectT = TypeVar("_SelectT", bound="Select")
7580

7681

7782
def is_regional(region_id: UUID, region_index: int) -> bool:
@@ -566,8 +571,8 @@ def loop_position(
566571

567572

568573
def build_loop_family_legs(
569-
query: Select[Any], family_member: DynamicRegion, *, iteration: int | None = None
570-
) -> tuple[Select[Any], Select[Any]]:
574+
query: _SelectT, family_member: DynamicRegion, *, iteration: int | None = None
575+
) -> tuple[_SelectT, _SelectT]:
571576
"""
572577
Split ``query``, a selection from ``task_instance``, into the rows that belong to one loop family.
573578
@@ -648,7 +653,7 @@ def _match_any(column, value: str | Collection[str]) -> ColumnElement[bool]:
648653

649654

650655
def _filter_producers(
651-
query: Select,
656+
query: _SelectT,
652657
*,
653658
dag_id: str,
654659
run_id: str,
@@ -658,7 +663,7 @@ def _filter_producers(
658663
region_id: UUID | None,
659664
region_index: int | None,
660665
top_level_only: bool,
661-
) -> Select:
666+
) -> _SelectT:
662667
"""Push every coordinate predicate that does not need loop ancestry into SQL."""
663668
from airflow.models.taskinstance import TaskInstance
664669

@@ -707,7 +712,7 @@ def _validate_producer_request(
707712

708713

709714
def _build_candidate_query(
710-
query: Select[Any],
715+
query: _SelectT,
711716
*,
712717
dag_id: str,
713718
run_id: str,
@@ -718,7 +723,7 @@ def _build_candidate_query(
718723
region_id: UUID | None,
719724
region_index: int | None,
720725
session: Session,
721-
) -> Select[Any] | CompoundSelect | None:
726+
) -> _SelectT | CompoundSelect | None:
722727
"""
723728
Narrow ``query`` to the candidate producers of the caller's scope, or ``None`` if there are none.
724729
@@ -823,7 +828,7 @@ def select_current_producer_ids(
823828
region_id: UUID | None = None,
824829
region_index: int | None = None,
825830
session: Session,
826-
) -> Select[tuple[UUID]]:
831+
) -> UUIDSelect:
827832
"""
828833
Select the ids of the live producers :func:`resolve_current_producers` would return.
829834
@@ -928,7 +933,7 @@ def _build_pass_ownership_filter(
928933

929934
def select_loop_producer_ids(
930935
*, dag_id: str, run_id: str, loop_node_id: str, task_id: str, is_mapped: bool, session: Session
931-
) -> Select[tuple[UUID]]:
936+
) -> UUIDSelect:
932937
"""
933938
Select every live execution of ``task_id`` across all passes of the loop ``loop_node_id``.
934939

‎airflow-core/src/airflow/models/pool.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -398,4 +398,4 @@ def get_name_to_team_name_mapping(
398398
pool_names: list[str], *, session: Session = NEW_SESSION
399399
) -> dict[str, str | None]:
400400
stmt = select(Pool.pool, Pool.team_name).where(Pool.pool.in_(pool_names))
401-
return {pool: team_name for pool, team_name in session.execute(stmt)}
401+
return {pool: team_name for pool, team_name in session.execute(stmt) if pool is not None}

‎airflow-core/src/airflow/models/task_coordinates.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
from sqlalchemy.sql.elements import ColumnElement
5050

5151
from airflow.serialization.definitions.dag import SerializedDAG, SerializedOperator
52+
from airflow.utils.sqlalchemy import UUIDSelect
5253

5354

5455
LOOP_GATE_OPERATOR = "LoopGateOperator"
@@ -361,7 +362,7 @@ def select_legacy_task_ids(
361362
run_id: str,
362363
task_ids: Collection[str],
363364
slots: Collection[tuple[str, int]],
364-
) -> Select[tuple[UUID]]:
365+
) -> UUIDSelect:
365366
"""Select, in one statement, the region-less task instances that the named tasks or slots address."""
366367
targets = []
367368
if task_ids:
@@ -730,7 +731,7 @@ def resolve(
730731

731732
def select_skip_target_ids(
732733
self, *, caller: TaskInstance, task_id: str, map_indexes: int | None = None
733-
) -> Select[tuple[UUID]]:
734+
) -> UUIDSelect:
734735
"""
735736
Select the task instance ids a skip request from ``caller`` reaches for ``task_id``.
736737
@@ -765,7 +766,7 @@ def select_skip_target_ids(
765766
map_indexes=map_indexes,
766767
)
767768

768-
def _select_loop_producer_ids(self, request: _ProducerRequest) -> Select[tuple[UUID]]:
769+
def _select_loop_producer_ids(self, request: _ProducerRequest) -> UUIDSelect:
769770
if TYPE_CHECKING:
770771
assert request.loop_node_id is not None
771772
return select_loop_producer_ids(
@@ -789,7 +790,7 @@ def select_producer_ids(
789790
map_indexes: int | Collection[int] | None = None,
790791
previous_iteration: bool = False,
791792
all_iterations: bool = False,
792-
) -> Select[tuple[UUID]]:
793+
) -> UUIDSelect:
793794
"""Select the task instance ids :meth:`resolve` returns, without loading the producers."""
794795
if region_index is not None and region_id is None:
795796
raise ValueError("region_index requires an explicit producer region_id")

‎airflow-core/src/airflow/models/xcom.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,9 @@
6868
from airflow.models.dagrun import DagRun
6969
from airflow.models.taskinstance import TaskInstance
7070

71+
# TODO: use Select[XComModel] once the minimum SQLAlchemy is 2.1; 2.0 uses Select[tuple[XComModel]].
72+
XComModelSelect = Select
73+
7174

7275
XCOM_RETURN_KEY = "return_value"
7376

@@ -328,7 +331,7 @@ def get_many(
328331
include_prior_dates: bool = False,
329332
limit: int | None = None,
330333
try_number: int | None = None,
331-
) -> Select[tuple[XComModel]]:
334+
) -> XComModelSelect:
332335
"""
333336
Composes a query to get one or more XCom entries.
334337
@@ -596,7 +599,7 @@ def _reject_projection_write(mapper, connection, target):
596599
raise TypeError("XCom read projection is read-only")
597600

598601

599-
def build_xcom_read_query(*, producer_ids: Select, key: str | None = None) -> Select[tuple[XComModel]]:
602+
def build_xcom_read_query(*, producer_ids: Select, key: str | None = None) -> XComModelSelect:
600603
model = get_xcom_model()
601604
entity = aliased(model, _filter_rows(model.__table__, producer_ids=producer_ids, key=key))
602605
return select(entity).execution_options(populate_existing=True)

‎airflow-core/src/airflow/utils/sqlalchemy.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@
5252

5353
from airflow.typing_compat import Self
5454

55+
# TODO: use Select[UUID] once the minimum SQLAlchemy is 2.1; 2.0 uses Select[tuple[UUID]].
56+
UUIDSelect = Select
57+
5558

5659
log = logging.getLogger(__name__)
5760

‎airflow-core/tests/unit/api_fastapi/auth/managers/test_base_auth_manager.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
from airflow.api_fastapi.auth.tokens import JWTGenerator, JWTValidator
3939
from airflow.api_fastapi.common.types import MenuItem
4040
from airflow.exceptions import RemovedInAirflow4Warning
41+
from airflow.models.pool import Pool
4142
from airflow.models.team import Team
4243

4344
from tests_common.test_utils.config import conf_vars
@@ -799,6 +800,22 @@ def side_effect_func(
799800
result = auth_manager.get_authorized_pools(user=user, session=session)
800801
assert result == expected
801802

803+
@pytest.mark.db_test
804+
def test_get_authorized_pools_skips_null_names(self, auth_manager, session):
805+
session.add_all(
806+
[
807+
Pool(pool=None, slots=1, include_deferred=False),
808+
Pool(pool="authorized_pool", slots=1, include_deferred=False),
809+
]
810+
)
811+
session.flush()
812+
auth_manager.is_authorized_pool = create_autospec(auth_manager.is_authorized_pool, return_value=True)
813+
814+
result = auth_manager.get_authorized_pools(user=Mock(spec=BaseAuthManagerUserTest), session=session)
815+
816+
assert None not in result
817+
assert "authorized_pool" in result
818+
802819
@pytest.mark.parametrize(
803820
("authorized_uri_prefix", "rows", "expected"),
804821
[

0 commit comments

Comments
 (0)