Skip to content

Commit a53b3bc

Browse files
committed
Propagate stale data errors to the scheduler retry boundary
ORM flush failures can leave the Session unusable just like driver errors. Keeping both retryable error categories visible to the owning transaction lets its existing rollback and retry policy recover the batch instead of masking the original failure.
1 parent 4f621b3 commit a53b3bc

2 files changed

Lines changed: 21 additions & 7 deletions

File tree

‎airflow-core/src/airflow/jobs/scheduler_job_runner.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
)
5353
from sqlalchemy.exc import DBAPIError, OperationalError
5454
from sqlalchemy.orm import joinedload, lazyload, load_only, make_transient, selectinload
55+
from sqlalchemy.orm.exc import StaleDataError
5556
from sqlalchemy.sql import expression
5657

5758
from airflow import settings
@@ -418,7 +419,7 @@ def _get_current_dag(self, dag_id: str, session: Session) -> SerializedDAG | Non
418419
return None
419420
serdag.load_op_links = False
420421
return serdag.dag
421-
except DBAPIError:
422+
except (DBAPIError, StaleDataError):
422423
raise
423424
except Exception:
424425
self.log.exception("Failed to deserialize DAG '%s'", dag_id)
@@ -2841,7 +2842,7 @@ def _create_dag_runs(self, dag_models: Collection[DagModel], session: Session) -
28412842
# DagModel.create_dagrun() when dag is misconfigured. The scheduler should not
28422843
# crash due to misconfigured Dags. We should log non-database exceptions
28432844
# and continue to the next serdag.
2844-
except DBAPIError:
2845+
except (DBAPIError, StaleDataError):
28452846
# The owning transaction must roll back and re-query candidates with HA locks.
28462847
# Persistent per-Dag failure isolation remains tracked in
28472848
# https://github.com/apache/airflow/issues/59120.

‎airflow-core/tests/unit/jobs/test_scheduler_job.py‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
from sqlalchemy.dialects import mysql
4545
from sqlalchemy.exc import DBAPIError, IntegrityError
4646
from sqlalchemy.orm import joinedload
47+
from sqlalchemy.orm.exc import StaleDataError
4748

4849
from airflow import settings
4950
from airflow._shared.module_loading import qualname
@@ -6572,7 +6573,10 @@ def test_create_dag_runs(self, dag_maker):
65726573

65736574
@pytest.mark.parametrize("failure_point", ["deserialize", "create"])
65746575
@pytest.mark.parametrize("failures", [1, MAX_DB_RETRIES])
6575-
def test_create_dagruns_retries_original_db_error(self, failure_point, failures, dag_maker, session):
6576+
@pytest.mark.parametrize("error_type", [IntegrityError, StaleDataError])
6577+
def test_create_dagruns_retries_original_db_error(
6578+
self, failure_point, failures, error_type, dag_maker, session
6579+
):
65766580
with dag_maker(dag_id="retry_db_error", schedule="@once", session=session):
65776581
EmptyOperator(task_id="task")
65786582

@@ -6594,10 +6598,18 @@ def fail_then_call(*args, **kwargs):
65946598
calls += 1
65956599
try:
65966600
if calls <= failures:
6597-
kwargs["session"].add(Pool(pool=pool_name, slots=1, include_deferred=False))
6598-
kwargs["session"].flush()
6601+
failing_session = kwargs["session"]
6602+
if error_type is IntegrityError:
6603+
failing_session.add(Pool(pool=pool_name, slots=1, include_deferred=False))
6604+
else:
6605+
pool = failing_session.scalars(select(Pool).where(Pool.pool == pool_name)).one()
6606+
pool.slots += 1
6607+
# Bypass ORM synchronization so the pending UPDATE matches no rows.
6608+
failing_session.connection().execute(delete(Pool).where(Pool.id == pool.id))
6609+
failing_session.flush()
65996610
return original(*args, **kwargs)
6600-
except IntegrityError as error:
6611+
except error_type as error:
6612+
assert not kwargs["session"].is_active
66016613
errors.append(error)
66026614
raise
66036615

@@ -6607,7 +6619,7 @@ def fail_then_call(*args, **kwargs):
66076619
prohibit_commit(session) as guard,
66086620
):
66096621
if failures == MAX_DB_RETRIES:
6610-
with pytest.raises(IntegrityError) as raised:
6622+
with pytest.raises(error_type) as raised:
66116623
self.job_runner._create_dagruns_for_dags(guard, session)
66126624
assert raised.value is errors[-1]
66136625
assert session.is_active
@@ -6619,6 +6631,7 @@ def fail_then_call(*args, **kwargs):
66196631
assert runs[0].state == DagRunState.QUEUED
66206632
assert calls == min(failures + 1, MAX_DB_RETRIES)
66216633
assert len(errors) == failures
6634+
assert session.scalars(select(Pool).where(Pool.pool == pool_name)).one().slots == 1
66226635

66236636
@pytest.mark.backend("mysql")
66246637
@pytest.mark.parametrize("autoflush", [False, True])

0 commit comments

Comments
 (0)