Skip to content

Commit b3cffc4

Browse files
authored
Use atomic upsert for asset run queue on MySQL to avoid deadlocks (#69977)
Under concurrent asset-event fan-out, the per-row SAVEPOINT loop used for non-PostgreSQL backends deadlocks on MySQL/InnoDB and surfaces confusing "SAVEPOINT ... does not exist" errors, and the deadlock is not retried. Route MySQL to a single-statement INSERT ... ON DUPLICATE KEY UPDATE, mirroring the existing PostgreSQL path, so the enqueue is atomic and holds locks far more briefly. closes: #69938
1 parent 8037e9a commit b3cffc4

2 files changed

Lines changed: 65 additions & 5 deletions

File tree

‎airflow-core/src/airflow/assets/manager.py‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -518,12 +518,17 @@ def _queue_dagruns(
518518
# mapped) tasks update the same asset, this can fail with a unique
519519
# constraint violation.
520520
#
521-
# If we support it, use ON CONFLICT to do nothing, otherwise
522-
# "fallback" to running this in a nested transaction. This is needed
523-
# so that the adding of these rows happens in the same transaction
524-
# where `ti.state` is changed.
525-
if get_dialect_name(session) == "postgresql":
521+
# Where the dialect supports a single-statement "insert, ignore on
522+
# conflict" we use it; it is atomic, avoids the per-row SAVEPOINT churn,
523+
# and holds locks for far less time (which on MySQL/InnoDB also makes the
524+
# concurrent fan-out much less deadlock-prone). Otherwise we "fallback" to
525+
# a nested transaction per row. Either way the rows are added in the same
526+
# transaction where `ti.state` is changed.
527+
dialect_name = get_dialect_name(session)
528+
if dialect_name == "postgresql":
526529
return cls._queue_dagruns_nonpartitioned_postgres(asset_id, non_partitioned_dags, session)
530+
if dialect_name == "mysql":
531+
return cls._queue_dagruns_nonpartitioned_mysql(asset_id, non_partitioned_dags, session)
527532
return cls._queue_dagruns_nonpartitioned_slow_path(asset_id, non_partitioned_dags, session)
528533

529534
@classmethod
@@ -817,6 +822,20 @@ def _queue_dagruns_nonpartitioned_postgres(
817822
stmt = insert(AssetDagRunQueue).values(asset_id=asset_id).on_conflict_do_nothing()
818823
session.execute(stmt, values)
819824

825+
@classmethod
826+
def _queue_dagruns_nonpartitioned_mysql(
827+
cls, asset_id: int, dags_to_queue: set[DagModel], session: Session
828+
) -> None:
829+
from sqlalchemy.dialects.mysql import insert
830+
831+
values = [{"target_dag_id": dag.dag_id} for dag in dags_to_queue]
832+
stmt = insert(AssetDagRunQueue).values(asset_id=asset_id)
833+
# MySQL has no "ON CONFLICT DO NOTHING"; a no-op ON DUPLICATE KEY UPDATE turns a
834+
# conflicting (asset_id, target_dag_id) row into a no-op rather than an error,
835+
# matching the Postgres path.
836+
stmt = stmt.on_duplicate_key_update(target_dag_id=stmt.inserted.target_dag_id)
837+
session.execute(stmt, values)
838+
820839

821840
def resolve_asset_manager() -> AssetManager:
822841
"""Retrieve the asset manager."""

‎airflow-core/tests/unit/assets/test_manager.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
import pytest
2828
from sqlalchemy import delete, func, select
29+
from sqlalchemy.dialects import mysql
2930
from sqlalchemy.orm import Session
3031

3132
from airflow import settings
@@ -211,6 +212,46 @@ def test_register_asset_change_no_downstreams(self, session, mock_task_instance)
211212
)
212213
assert session.scalar(select(func.count()).select_from(AssetDagRunQueue)) == 0
213214

215+
@pytest.mark.parametrize(
216+
("dialect_name", "expected_helper"),
217+
[
218+
("postgresql", "_queue_dagruns_nonpartitioned_postgres"),
219+
("mysql", "_queue_dagruns_nonpartitioned_mysql"),
220+
("sqlite", "_queue_dagruns_nonpartitioned_slow_path"),
221+
],
222+
)
223+
def test_queue_dagruns_routes_by_dialect(self, dialect_name, expected_helper):
224+
"""Test that _queue_dagruns routes to the dialect-appropriate queue helper."""
225+
dag = DagModel(dag_id="dag1")
226+
session = mock.MagicMock(spec=Session)
227+
with (
228+
mock.patch("airflow.assets.manager.get_dialect_name", return_value=dialect_name),
229+
mock.patch.object(AssetManager, "_queue_partitioned_dags"),
230+
mock.patch.object(AssetManager, expected_helper) as mock_helper,
231+
):
232+
AssetManager._queue_dagruns(
233+
asset_id=1,
234+
dags_to_queue={dag},
235+
partition_key=None,
236+
partition_date=None,
237+
event=mock.MagicMock(),
238+
task_instance=None,
239+
session=session,
240+
)
241+
mock_helper.assert_called_once_with(1, {dag}, session)
242+
243+
def test_queue_dagruns_nonpartitioned_mysql_builds_upsert(self):
244+
"""Test that the MySQL queue path emits an INSERT ... ON DUPLICATE KEY UPDATE."""
245+
dag = DagModel(dag_id="dag1")
246+
session = mock.MagicMock(spec=Session)
247+
248+
AssetManager._queue_dagruns_nonpartitioned_mysql(asset_id=1, dags_to_queue={dag}, session=session)
249+
250+
stmt, values = session.execute.call_args.args
251+
compiled = str(stmt.compile(dialect=mysql.dialect())).upper()
252+
assert "ON DUPLICATE KEY UPDATE" in compiled
253+
assert values == [{"target_dag_id": "dag1"}]
254+
214255
def test_register_asset_change_notifies_asset_listener(
215256
self, session, mock_task_instance, testing_dag_bundle, listener_manager
216257
):

0 commit comments

Comments
 (0)