Skip to content

Commit 8395c29

Browse files
authored
Fix scheduler orphaned task reset logging crash (#67822)
The scheduler orphaned-task adoption path loaded task instances with a narrow load_only(...) set and then called repr(ti) while building the reset log message. TaskInstance.__repr__ reads map_index and state, so detached instances with deferred columns could raise DetachedInstanceError and crash the scheduler on restart.
1 parent 201acda commit 8395c29

4 files changed

Lines changed: 102 additions & 7 deletions

File tree

‎airflow-core/src/airflow/jobs/scheduler_job_runner.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2851,7 +2851,17 @@ def adopt_or_reset_orphaned_tasks(self, *, session: Session = NEW_SESSION) -> in
28512851
.where(Job.state.is_distinct_from(JobState.RUNNING))
28522852
.join(TI.dag_run)
28532853
.where(DagRun.state == DagRunState.RUNNING)
2854-
.options(load_only(TI.dag_id, TI.task_id, TI.run_id, TI.external_executor_id))
2854+
.options(
2855+
load_only(
2856+
TI.id,
2857+
TI.dag_id,
2858+
TI.task_id,
2859+
TI.run_id,
2860+
TI.map_index,
2861+
TI.state,
2862+
TI.external_executor_id,
2863+
)
2864+
)
28552865
)
28562866

28572867
# Lock these rows, so that another scheduler can't try and adopt these too

‎airflow-core/src/airflow/models/taskinstance.py‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@
6666
from sqlalchemy.ext.mutable import MutableDict
6767
from sqlalchemy.orm import Mapped, lazyload, mapped_column, reconstructor, relationship
6868
from sqlalchemy.orm.attributes import NO_VALUE, set_committed_value
69+
from sqlalchemy.orm.exc import DetachedInstanceError, ObjectDeletedError
6970

7071
from airflow import settings
7172
from airflow._shared.observability.metrics import stats
@@ -1158,10 +1159,22 @@ def get_failed_dep_statuses(
11581159
yield dep_status
11591160

11601161
def __repr__(self) -> str:
1161-
prefix = f"<TaskInstance: {self.dag_id}.{self.task_id} {self.run_id} "
1162-
if self.map_index != -1:
1163-
prefix += f"map_index={self.map_index} "
1164-
return prefix + f"[{self.state}] ti_id={self.id}>"
1162+
# ``__repr__`` is used in logging and must never raise. Real values are printed
1163+
# whenever they can be read (including a normal lazy-load on an *attached* instance);
1164+
# we only fall back to a placeholder when SQLAlchemy cannot produce the value at all:
1165+
# a deferred column on a *detached* instance (DetachedInstanceError), or a row deleted
1166+
# out from under an expired instance (ObjectDeletedError).
1167+
def field(name: str) -> Any:
1168+
try:
1169+
return getattr(self, name)
1170+
except (DetachedInstanceError, ObjectDeletedError):
1171+
return "<deferred>"
1172+
1173+
prefix = f"<TaskInstance: {field('dag_id')}.{field('task_id')} {field('run_id')} "
1174+
map_index = field("map_index")
1175+
if map_index != -1:
1176+
prefix += f"map_index={map_index} "
1177+
return prefix + f"[{field('state')}] ti_id={field('id')}>"
11651178

11661179
def next_retry_datetime(self):
11671180
"""

‎airflow-core/tests/unit/jobs/test_scheduler_job.py‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
import psutil
3737
import pytest
3838
import time_machine
39-
from sqlalchemy import delete, func, select, update
39+
from sqlalchemy import delete, func, inspect, select, update
4040
from sqlalchemy.dialects import mysql
4141
from sqlalchemy.orm import joinedload
4242

@@ -3297,6 +3297,40 @@ def test_adopt_or_reset_orphaned_tasks(self, dag_maker, session):
32973297
ti2 = dr2.get_task_instance(task_id=op1.task_id, session=session)
32983298
assert ti2.state == State.NONE, "Tasks run by Backfill Jobs should be treated the same"
32993299

3300+
def test_adopt_or_reset_orphaned_tasks_loads_state_for_reset_logging(
3301+
self, dag_maker, session, mock_executor
3302+
):
3303+
with dag_maker("test_adopt_or_reset_orphaned_tasks_loads_state_for_reset_logging", session=session):
3304+
op1 = EmptyOperator(task_id="op1")
3305+
3306+
scheduler_job = Job()
3307+
session.add(scheduler_job)
3308+
session.flush()
3309+
3310+
dr = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED)
3311+
ti = dr.get_task_instance(task_id=op1.task_id, session=session)
3312+
ti.state = State.QUEUED
3313+
ti.queued_by_job_id = scheduler_job.id
3314+
session.commit()
3315+
session.expunge_all()
3316+
3317+
def refuse_adoption(tis):
3318+
assert len(tis) == 1
3319+
# ``repr(ti)`` in the reset path reads both ``state`` and ``map_index``; the query
3320+
# must load both so the reset log stays accurate (and never lazy-loads on detach).
3321+
unloaded = inspect(tis[0]).unloaded
3322+
assert "state" not in unloaded
3323+
assert "map_index" not in unloaded
3324+
# repr must render the real state, not the ``<deferred>`` fallback.
3325+
assert "queued" in repr(tis[0])
3326+
return tis
3327+
3328+
mock_executor.try_adopt_task_instances.side_effect = refuse_adoption
3329+
3330+
self.job_runner = SchedulerJobRunner(job=Job(), num_runs=0)
3331+
3332+
assert self.job_runner.adopt_or_reset_orphaned_tasks(session=session) == 1
3333+
33003334
def test_adopt_or_reset_orphaned_tasks_multiple_executors(self, dag_maker, mock_executors):
33013335
"""
33023336
Test that with multiple executors configured tasks are sorted correctly and handed off to the

‎airflow-core/tests/unit/models/test_taskinstance.py‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,9 @@
3333
from opentelemetry import trace
3434
from opentelemetry.sdk.trace import TracerProvider
3535
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
36-
from sqlalchemy import delete, func, select
36+
from sqlalchemy import delete, func, inspect as sa_inspect, select
3737
from sqlalchemy.exc import IntegrityError
38+
from sqlalchemy.orm import load_only
3839
from sqlalchemy.orm.attributes import set_committed_value
3940

4041
from airflow import settings
@@ -3897,3 +3898,40 @@ def test_clear_task_instances_preserves_detail_level(dag_maker, session):
38973898
new_ctx = TraceContextTextMapPropagator().extract(dag_run.context_carrier)
38983899
span = trace.get_current_span(new_ctx)
38993900
assert get_task_span_detail_level(span) == 2
3901+
3902+
3903+
@pytest.mark.db_test
3904+
def test_task_instance_repr_does_not_raise_for_deferred_columns(dag_maker, session):
3905+
"""``TaskInstance.__repr__`` must survive *any* deferred column it reads.
3906+
3907+
Regression test for issue #67813: the scheduler's orphaned-task adoption loaded
3908+
TaskInstances via ``load_only`` and then called ``repr(ti)`` on detached instances.
3909+
``__repr__`` reads ``map_index`` and ``state`` (among others); on a detached instance
3910+
a column that was not loaded raises ``DetachedInstanceError``. ``__repr__`` is used in
3911+
logging and must degrade gracefully — printing ``<deferred>`` — instead of crashing.
3912+
"""
3913+
with dag_maker("test_repr_deferred_columns", session=session):
3914+
EmptyOperator(task_id="op1")
3915+
dr = dag_maker.create_dagrun()
3916+
ti = dr.get_task_instance(task_id="op1", session=session)
3917+
ti.state = State.QUEUED
3918+
session.commit()
3919+
ti_id = ti.id
3920+
3921+
# Reload the row with ``map_index`` and ``state`` left as deferred columns, then detach
3922+
# the instance so that touching them would otherwise require a (now-impossible) DB load.
3923+
session.expunge_all()
3924+
reloaded = session.scalar(
3925+
select(TaskInstance)
3926+
.where(TaskInstance.id == ti_id)
3927+
.options(load_only(TaskInstance.dag_id, TaskInstance.task_id, TaskInstance.run_id))
3928+
)
3929+
session.expunge(reloaded)
3930+
unloaded = sa_inspect(reloaded).unloaded
3931+
assert "map_index" in unloaded
3932+
assert "state" in unloaded
3933+
3934+
result = repr(reloaded) # would raise DetachedInstanceError without the guard
3935+
3936+
assert "<deferred>" in result
3937+
assert "[queued]" not in result

0 commit comments

Comments
 (0)