Skip to content

Commit e09ad45

Browse files
Refresh bundle version in place when DAG serialization is unchanged (#68336)
* Refresh bundle version in place when DAG serialization is unchanged When a DAG bundle advances to a new version/commit but the DAG's serialized content is byte-identical, write_dag took the 'unchanged' fast path and left the latest DagVersion's bundle_version pinned to the old commit. Tasks resolve their code from ti.dag_version.bundle_version at run time, so new runs and reruns 'with the latest bundle version' executed an outdated commit. Refresh the latest version's bundle_version/version_data (and dag code) in place instead. This does not create a new DagVersion, avoiding version inflation on every bundle commit, while keeping the recorded bundle pointer current. closes: #67657 * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <[email protected]> --------- Co-authored-by: Copilot Autofix powered by AI <[email protected]>
1 parent 14ab6a0 commit e09ad45

2 files changed

Lines changed: 78 additions & 3 deletions

File tree

‎airflow-core/src/airflow/models/serialized_dag.py‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -687,9 +687,21 @@ def write_dag(
687687
and dag_version
688688
and dag_version.bundle_name == bundle_name
689689
):
690-
if name_updated:
691-
# The serialized DAG itself is unchanged, but deadline alert name(s) were
692-
# updated in the DB, so report True so callers know a write did occur.
690+
# Serialized content is unchanged, so we don't create a new DagVersion.
691+
# But if the bundle advanced, refresh the latest version's pointer in place — tasks resolve
692+
# their code from ``ti.dag_version.bundle_version`` at run time, so a stale
693+
# pointer makes runs execute an outdated commit.
694+
bundle_metadata_changed = (
695+
dag_version.bundle_version != bundle_version or dag_version.version_data != version_data
696+
)
697+
if bundle_metadata_changed:
698+
dag_version.bundle_version = bundle_version
699+
dag_version.version_data = version_data
700+
session.merge(dag_version)
701+
DagCode.update_source_code(dag_id=dag.dag_id, fileloc=dag.fileloc, session=session)
702+
if name_updated or bundle_metadata_changed:
703+
# A write occurred — a deadline alert name update and/or a bundle
704+
# metadata refresh — so report True so callers know the DB changed.
693705
return True
694706
log.debug("Serialized DAG (%s) is unchanged. Skipping writing to DB", dag.dag_id)
695707
return False

‎airflow-core/tests/unit/models/test_serialized_dag.py‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -585,6 +585,69 @@ def test_new_dag_version_created_when_bundle_name_changes_and_hash_unchanged(sel
585585
# There should now be two versions of the DAG
586586
assert session.scalar(select(func.count()).select_from(DagVersion)) == 2
587587

588+
def test_bundle_version_refreshed_in_place_when_hash_unchanged(self, dag_maker, session):
589+
"""When the bundle advances to a new version/commit but the DAG's serialized
590+
content is unchanged, ``write_dag`` must refresh the latest DagVersion's
591+
``bundle_version`` in place (so tasks resolve the current commit) without
592+
creating a new DagVersion (which would inflate versions on every commit).
593+
"""
594+
with dag_maker("test_dag_bundle_version_refresh", bundle_name="bundleA") as dag:
595+
EmptyOperator(task_id="task1")
596+
# Pin the version with task instances so the in-place "no TI" branch is NOT taken.
597+
dag_maker.create_dagrun(run_id="test_run")
598+
599+
did_write = SDM.write_dag(
600+
LazyDeserializedDAG.from_dag(dag),
601+
bundle_name="bundleA",
602+
bundle_version="commit_A",
603+
version_data={"manifest": "A"},
604+
session=session,
605+
)
606+
assert did_write is True
607+
assert session.scalar(select(func.count()).select_from(DagVersion)) == 1
608+
latest = DagVersion.get_latest_version(dag.dag_id, session=session)
609+
assert latest.bundle_version == "commit_A"
610+
assert latest.version_data == {"manifest": "A"}
611+
612+
# Same content, same bundle_name, but the bundle moved to a new commit.
613+
did_write = SDM.write_dag(
614+
LazyDeserializedDAG.from_dag(dag),
615+
bundle_name="bundleA",
616+
bundle_version="commit_B",
617+
version_data={"manifest": "B"},
618+
session=session,
619+
)
620+
621+
# No new version was created, but the latest version's bundle pointer advanced.
622+
assert did_write is True
623+
assert session.scalar(select(func.count()).select_from(DagVersion)) == 1
624+
latest = DagVersion.get_latest_version(dag.dag_id, session=session)
625+
assert latest.bundle_version == "commit_B"
626+
assert latest.version_data == {"manifest": "B"}
627+
628+
def test_write_dag_unchanged_with_same_bundle_version_skips_write(self, dag_maker, session):
629+
"""A re-parse with identical content and identical bundle metadata is a no-op."""
630+
with dag_maker("test_dag_unchanged_noop", bundle_name="bundleA") as dag:
631+
EmptyOperator(task_id="task1")
632+
dag_maker.create_dagrun(run_id="test_run")
633+
634+
SDM.write_dag(
635+
LazyDeserializedDAG.from_dag(dag),
636+
bundle_name="bundleA",
637+
bundle_version="commit_A",
638+
session=session,
639+
)
640+
641+
did_write = SDM.write_dag(
642+
LazyDeserializedDAG.from_dag(dag),
643+
bundle_name="bundleA",
644+
bundle_version="commit_A",
645+
session=session,
646+
)
647+
648+
assert did_write is False
649+
assert session.scalar(select(func.count()).select_from(DagVersion)) == 1
650+
588651
def test_hash_method_removes_fileloc_and_remains_consistent(self):
589652
"""Test that the hash method removes fileloc before hashing."""
590653
test_data = {

0 commit comments

Comments
 (0)