Skip to content

Commit eaf5fc1

Browse files
allow deadline alert UUID references in serialized Dag schema (#70148)
Co-authored-by: Kevin Yang <[email protected]>
1 parent f5ec361 commit eaf5fc1

4 files changed

Lines changed: 115 additions & 1 deletion

File tree

‎airflow-core/src/airflow/models/serialized_dag.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
from __future__ import annotations
2121

22+
import copy
2223
import logging
2324
import zlib
2425
from collections.abc import Callable, Iterable, Iterator, Sequence
@@ -647,6 +648,11 @@ def write_dag(
647648
name_updated = False
648649
reused_deadline_data: dict[str, dict] | None = None
649650
if dag.data.get("dag", {}).get("deadline"):
651+
# The deadline handling below rewrites data["dag"]["deadline"] from a list of
652+
# encoded dicts into a list of UUID references. Work on a copy so we never mutate
653+
# the caller's LazyDeserializedDAG in place.
654+
655+
dag = dag.model_copy(update={"data": copy.deepcopy(dag.data)})
650656
# Try to reuse existing deadline UUIDs if the deadline definitions haven't changed.
651657
# This preserves the hash and avoids unnecessary SerializedDagModel recreations.
652658
existing_serialized_dag = session.scalar(

‎airflow-core/src/airflow/serialization/schema.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,13 @@
214214
"type": "array",
215215
"items": { "$ref": "#/definitions/dict" }
216216
},
217+
{
218+
"$comment": "Once persisted, a Dag's deadline alerts live as rows in the deadline_alert table and the serialized Dag keeps only a list of UUID strings referencing them (see SerializedDagModel._generate_deadline_uuids). This branch lets the stored form validate at any lifecycle stage, not only before the dict->UUID rewrite.",
219+
"type": "array",
220+
"items": { "type": "string",
221+
"pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"
222+
}
223+
},
217224
{ "type": "null" }
218225
]
219226
},

‎airflow-core/tests/unit/models/test_serialized_dag.py‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
from __future__ import annotations
2121

22+
import copy
2223
import logging
2324
from datetime import timedelta
2425
from unittest import mock
@@ -1234,3 +1235,99 @@ def test_deadline_reuse_skips_write_when_hash_matches(self, testing_dag_bundle,
12341235
alert = session.scalar(select(DAM).where(DAM.serialized_dag_id == orig_serdag.id))
12351236
assert alert is not None
12361237
assert alert.id == orig_alert.id
1238+
1239+
def test_write_dag_with_deadline_passes_schema_validation(self, testing_dag_bundle, session):
1240+
"""The persisted serialized Dag for a deadline-bearing Dag must satisfy the JSON schema.
1241+
1242+
write_dag stores ``data["dag"]["deadline"]`` as a list of UUID strings referencing
1243+
deadline_alert rows, so the schema has to accept that persisted form and not only the
1244+
list-of-dicts form produced before the dict->UUID rewrite.
1245+
"""
1246+
dag_id = "test_deadline_schema_valid"
1247+
dag = DAG(
1248+
dag_id=dag_id,
1249+
deadline=DeadlineAlert(
1250+
reference=DeadlineReference.DAGRUN_QUEUED_AT,
1251+
interval=timedelta(minutes=5),
1252+
callback=AsyncCallback(empty_callback_for_deadline),
1253+
),
1254+
)
1255+
EmptyOperator(task_id="task1", dag=dag)
1256+
sync_dag_to_db(dag, session=session)
1257+
session.commit()
1258+
1259+
result = session.scalar(select(SDM).where(SDM.dag_id == dag_id))
1260+
persisted_deadline = result.data["dag"]["deadline"]
1261+
assert isinstance(persisted_deadline, list)
1262+
assert persisted_deadline
1263+
assert all(isinstance(ref, str) for ref in persisted_deadline)
1264+
1265+
# Must not raise: the stored UUID-reference form has to satisfy the serialized Dag schema.
1266+
DagSerialization.validate_schema(result.data)
1267+
1268+
def test_write_dag_does_not_mutate_caller_deadline_data(self, testing_dag_bundle, session):
1269+
"""write_dag must not rewrite the caller's LazyDeserializedDAG deadline in place.
1270+
1271+
The dict->UUID replacement in ``_generate_deadline_uuids`` has to happen on a copy so a
1272+
LazyDeserializedDAG the caller still references keeps its original list-of-dicts deadline.
1273+
"""
1274+
dag_id = "test_deadline_no_mutation"
1275+
dag = DAG(
1276+
dag_id=dag_id,
1277+
deadline=DeadlineAlert(
1278+
reference=DeadlineReference.DAGRUN_QUEUED_AT,
1279+
interval=timedelta(minutes=5),
1280+
callback=AsyncCallback(empty_callback_for_deadline),
1281+
),
1282+
)
1283+
EmptyOperator(task_id="task1", dag=dag)
1284+
sync_dag_to_db(dag, session=session)
1285+
session.commit()
1286+
1287+
# Change the interval so write_dag regenerates UUIDs (the dict->UUID rewrite path)
1288+
# rather than reusing the existing ones.
1289+
dag.deadline = DeadlineAlert(
1290+
reference=DeadlineReference.DAGRUN_QUEUED_AT,
1291+
interval=timedelta(minutes=10),
1292+
callback=AsyncCallback(empty_callback_for_deadline),
1293+
)
1294+
lazy_dag = LazyDeserializedDAG.from_dag(dag)
1295+
original_deadline = copy.deepcopy(lazy_dag.data["dag"]["deadline"])
1296+
assert original_deadline
1297+
assert all(isinstance(item, dict) for item in original_deadline)
1298+
1299+
SDM.write_dag(lazy_dag, bundle_name="testing", session=session)
1300+
session.commit()
1301+
1302+
assert lazy_dag.data["dag"]["deadline"] == original_deadline
1303+
1304+
def test_sync_dag_to_db_returns_db_normalized_deadline_ids(self, testing_dag_bundle, session):
1305+
"""sync_dag_to_db must return a SerializedDAG with the DB-normalized deadline UUIDs. Verify that the UUIDs returned by sync_dag_to_db match the persisted deadline_alert rows in the DB."""
1306+
dag_id = "test_sync_dag_to_db_deadline_ids"
1307+
dag = DAG(
1308+
dag_id=dag_id,
1309+
deadline=DeadlineAlert(
1310+
reference=DeadlineReference.DAGRUN_QUEUED_AT,
1311+
interval=timedelta(minutes=5),
1312+
callback=AsyncCallback(empty_callback_for_deadline),
1313+
),
1314+
)
1315+
EmptyOperator(task_id="task1", dag=dag)
1316+
1317+
scheduler_dag = sync_dag_to_db(dag, session=session)
1318+
session.commit()
1319+
1320+
latest_serdag = session.scalar(
1321+
select(SDM).where(SDM.dag_id == dag_id).order_by(SDM.created_at.desc())
1322+
)
1323+
assert latest_serdag is not None
1324+
1325+
persisted_alerts = session.scalars(select(DAM).where(DAM.serialized_dag_id == latest_serdag.id)).all()
1326+
1327+
persisted_uuids = {str(alert.id) for alert in persisted_alerts}
1328+
returned_uuids = scheduler_dag.deadline or []
1329+
1330+
assert returned_uuids
1331+
assert all(isinstance(ref, str) for ref in returned_uuids)
1332+
assert len(returned_uuids) == len(set(returned_uuids))
1333+
assert set(returned_uuids) == persisted_uuids

‎devel-common/src/tests_common/test_utils/dag.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,11 @@ def _write_dag(dag: DAG) -> SerializedDAG:
7373
SerializedDagModel.write_dag(
7474
LazyDeserializedDAG(data=data), bundle_name, bundle_version, session=session
7575
)
76-
return DagSerialization.from_dict(data)
76+
session.flush()
77+
serialized_dag = SerializedDagModel.get_dag(dag.dag_id, session=session)
78+
if serialized_dag is None:
79+
raise RuntimeError(f"Serialized DAG {dag.dag_id!r} was not found after writing to the database")
80+
return serialized_dag
7781

7882
SerializedDAG.bulk_write_to_db(bundle_name, bundle_version, dags, session=session)
7983
scheduler_dags = [_write_dag(dag) for dag in dags]

0 commit comments

Comments
 (0)