Skip to content

Commit 05981d3

Browse files
Fix task Clear and callback bundles after code-only Dag changes (#73915)
* Fix task Clear bundle selection after code-only Dag changes A deployed callable fix can leave serialization unchanged while an existing run stays pinned to an older bundle. Users need the latest-version clear option to remain available and an explicit selection to take effect for active runs. Scheduler callbacks also need to resolve the same bundle as task execution. * Pair task callback bundle manifest with the run's pinned version Callbacks and failure emails resolve their bundle version from the Dag run, but the manifest still came from the task instance's Dag version. When that version differs from the one the run was created with, a manifest-based bundle would initialize the run's bundle version from another version's manifest. Task execution and Dag-run callbacks already take both from the run's pinned version. * Run triggerer task callbacks on the run's pinned bundle A deferred task that finishes in the triggerer has its callback sent from there, and that path still took the bundle from the task instance's Dag version. A code-only update advances that version's bundle in place while the run stays pinned, so the callback would run newer code than the task did. Scheduler callbacks already resolve the bundle from the run. * Stop pairing a pinned bundle with an overwritten manifest A code-only update reuses the Dag version and overwrites its bundle version and manifest in place, while existing runs stay pinned to the older bundle version. Task execution and callbacks then shipped the run's older version with the newer manifest, so a manifest-based bundle would materialize code the run was never pinned to. Once the original manifest is gone, sending none is safer than sending one for a different version.
1 parent a8c55b7 commit 05981d3

10 files changed

Lines changed: 308 additions & 112 deletions

File tree

‎airflow-core/src/airflow/jobs/scheduler_job_runner.py‎

Lines changed: 7 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@
105105
ConnectionTestState,
106106
)
107107
from airflow.models.dag import DagModel
108-
from airflow.models.dag_version import DagVersion, _resolve_version_data
108+
from airflow.models.dag_version import DagVersion, _resolve_ti_callback_bundle_info
109109
from airflow.models.dagbag import CachedDBDagBag, DBDagBag
110110
from airflow.models.dagbundle import DagBundleModel
111111
from airflow.models.dagrun import DagRun
@@ -200,26 +200,6 @@ def _eager_load_dag_run_for_validation() -> tuple[LoaderOption, LoaderOption]:
200200
)
201201

202202

203-
def _resolve_ti_callback_bundle_info(ti: TaskInstance) -> tuple[str, str | None, Any]:
204-
"""
205-
Resolve the bundle name/version/version-data needed to build a TaskCallbackRequest or EmailRequest.
206-
207-
Used by the heartbeat-timeout purge path. Encapsulates the bundle-pinning semantics: fall back
208-
to ``dag_model`` for legacy tasks with no ``dag_version`` (pre-AIP-66 migrations), and leave the
209-
bundle version unpinned when the dag run itself wasn't pinned (``disable_bundle_versioning``),
210-
so the callback runs against the same code as the task did. ``process_executor_events`` inlines
211-
the same resolution for its externally-killed-task path.
212-
"""
213-
bundle_name = ti.dag_version.bundle_name if ti.dag_version else ti.dag_model.bundle_name
214-
bundle_version = (
215-
ti.dag_version.bundle_version
216-
if ti.dag_version and ti.dag_run.bundle_version is not None
217-
else ti.dag_run.bundle_version
218-
)
219-
version_data = _resolve_version_data(ti.dag_version, ti.dag_run.bundle_version)
220-
return bundle_name, bundle_version, version_data
221-
222-
223203
def _ensure_ti_has_dag_version_id(ti: TaskInstance, session: Session, log: Logger) -> bool:
224204
"""
225205
Ensure a TaskInstance has a valid dag_version_id for Pydantic serialisation.
@@ -1733,18 +1713,7 @@ def process_executor_events(
17331713
# Only log the error/extra info here, since the `ti.handle_failure()` path will log it
17341714
# too, which would lead to double logging
17351715
cls.logger().error(msg)
1736-
# Safely extract bundle info: prefer dag_version when available,
1737-
# fall back to dag_model/dag_run for legacy tasks migrated from
1738-
# Airflow 2 where dag_version may be None (AIP-66).
1739-
_bundle_name = ti.dag_version.bundle_name if ti.dag_version else ti.dag_model.bundle_name
1740-
# Mirror dag_run pinning: if the run wasn't pinned (e.g. dag.disable_bundle_versioning=True),
1741-
# leave the callback unpinned so it runs against the same code as the task.
1742-
_bundle_version = (
1743-
ti.dag_version.bundle_version
1744-
if ti.dag_version and ti.dag_run.bundle_version is not None
1745-
else ti.dag_run.bundle_version
1746-
)
1747-
_version_data = _resolve_version_data(ti.dag_version, ti.dag_run.bundle_version)
1716+
_bundle_name, _bundle_version, _version_data = _resolve_ti_callback_bundle_info(ti)
17481717
# Backfill dag_version_id for legacy tasks (Pydantic requires uuid.UUID).
17491718
if not _ensure_ti_has_dag_version_id(ti, session, cls.logger()):
17501719
continue
@@ -1778,15 +1747,9 @@ def process_executor_events(
17781747
"Sending email request for task %s to DAG Processor",
17791748
ti,
17801749
)
1781-
# Safely extract bundle info with fallback for legacy tasks
1782-
# (dag_version may be None after Airflow 2 → 3 migration).
1783-
_email_bundle_name = (
1784-
ti.dag_version.bundle_name if ti.dag_version else ti.dag_model.bundle_name
1785-
)
1786-
_email_bundle_version = (
1787-
ti.dag_version.bundle_version if ti.dag_version else ti.dag_run.bundle_version
1750+
_email_bundle_name, _email_bundle_version, _email_version_data = (
1751+
_resolve_ti_callback_bundle_info(ti)
17881752
)
1789-
_email_version_data = _resolve_version_data(ti.dag_version, ti.dag_run.bundle_version)
17901753
# Backfill dag_version_id for legacy tasks (Pydantic requires uuid.UUID).
17911754
if not _ensure_ti_has_dag_version_id(ti, session, cls.logger()):
17921755
continue
@@ -3496,19 +3459,9 @@ def _maybe_requeue_stuck_ti(self, *, ti, session, executor):
34963459
if task.has_on_failure_callback:
34973460
if inspect(ti).detached:
34983461
ti = session.merge(ti)
3499-
# Safely extract bundle info with fallback for legacy tasks
3500-
# (dag_version may be None after Airflow 2 → 3 migration).
3501-
_stuck_bundle_name = (
3502-
ti.dag_version.bundle_name if ti.dag_version else ti.dag_model.bundle_name
3462+
_stuck_bundle_name, _stuck_bundle_version, _stuck_version_data = (
3463+
_resolve_ti_callback_bundle_info(ti)
35033464
)
3504-
# Mirror dag_run pinning: if the run wasn't pinned (e.g. dag.disable_bundle_versioning=True),
3505-
# leave the callback unpinned so it runs against the same code as the task.
3506-
_stuck_bundle_version = (
3507-
ti.dag_version.bundle_version
3508-
if ti.dag_version and ti.dag_run.bundle_version is not None
3509-
else ti.dag_run.bundle_version
3510-
)
3511-
_stuck_version_data = _resolve_version_data(ti.dag_version, ti.dag_run.bundle_version)
35123465
# Backfill dag_version_id for legacy tasks (Pydantic requires uuid.UUID).
35133466
# Note: we cannot use `continue` here because this method is not
35143467
# inside a loop. If backfilling fails we simply skip the callback.
@@ -3980,6 +3933,7 @@ def _find_task_instances_without_heartbeats(self, *, session: Session) -> list[T
39803933
.options(selectinload(TI.dag_model))
39813934
.options(asset_loader)
39823935
.options(alias_loader)
3936+
.options(joinedload(TI.dag_run).selectinload(DagRun.created_dag_version))
39833937
.options(selectinload(TI.dag_version))
39843938
.with_hint(TI, "USE INDEX (ti_state)", dialect_name="mysql")
39853939
.join(DM, TI.dag_id == DM.dag_id)

‎airflow-core/src/airflow/models/dag_version.py‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
from sqlalchemy.sql import Select
4242

4343
from airflow.models.serialized_dag import SerializedDagModel
44+
from airflow.models.taskinstance import TaskInstance
4445

4546

4647
log = logging.getLogger(__name__)
@@ -416,7 +417,24 @@ def _resolve_version_data(
416417
"""Return a bundle version's ``version_data`` manifest, but only for pinned runs."""
417418
# Expose version_data only when the run is pinned (bundle_version set) and a DagVersion is
418419
# present, so the bundle initializes against the exact version the run used. Unpinned runs
419-
# follow the latest bundle state, and legacy rows have no DagVersion.
420-
if dag_version is not None and bundle_version is not None:
420+
# follow the latest bundle state, and legacy rows have no DagVersion. A code-only update
421+
# overwrites a reused DagVersion's bundle metadata in place, so its manifest describes the
422+
# pinned bundle only while the two bundle versions still match.
423+
if (
424+
dag_version is not None
425+
and bundle_version is not None
426+
and dag_version.bundle_version == bundle_version
427+
):
421428
return dag_version.version_data
422429
return None
430+
431+
432+
def _resolve_ti_callback_bundle_info(
433+
ti: TaskInstance,
434+
) -> tuple[str, str | None, dict[str, Any] | None]:
435+
"""Resolve bundle metadata for task callbacks and emails using the run's execution bundle."""
436+
bundle_name = ti.dag_version.bundle_name if ti.dag_version else ti.dag_model.bundle_name
437+
# A code-only update can advance the DagVersion's bundle while the run stays pinned.
438+
bundle_version = ti.dag_run.bundle_version
439+
version_data = _resolve_version_data(ti.dag_run.created_dag_version, ti.dag_run.bundle_version)
440+
return bundle_name, bundle_version, version_data

‎airflow-core/src/airflow/models/serialized_dag.py‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -705,10 +705,8 @@ def write_dag(
705705
new_dag_hash = cls.hash(dag.data)
706706

707707
if serialized_dag_hash == new_dag_hash and dag_version and dag_version.bundle_name == bundle_name:
708-
# Serialized content is unchanged, so we don't create a new DagVersion.
709-
# But if the bundle advanced, refresh the latest version's pointer in place — tasks resolve
710-
# their code from ``ti.dag_version.bundle_version`` at run time, so a stale
711-
# pointer makes runs execute an outdated commit.
708+
# Reuse the structural version with the latest bundle metadata. Existing runs
709+
# keep their execution bundle pinned separately in DagRun.bundle_version.
712710
bundle_metadata_changed = (
713711
dag_version.bundle_version != bundle_version or dag_version.version_data != version_data
714712
)

‎airflow-core/src/airflow/models/trigger.py‎

Lines changed: 3 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -722,24 +722,9 @@ def _submit_callback_if_necessary() -> None:
722722
):
723723
if task_instance.dag_model.relative_fileloc is None:
724724
raise RuntimeError("relative_fileloc should not be None for a finished task")
725-
from airflow.models.dag_version import _resolve_version_data
726-
727-
# Derive bundle identity from the TI's dag_version (falling back to dag_run/dag_model
728-
# for legacy/unpinned runs), mirroring the other callback sites so bundle_version and
729-
# version_data always describe the same version.
730-
bundle_name = (
731-
task_instance.dag_version.bundle_name
732-
if task_instance.dag_version
733-
else task_instance.dag_model.bundle_name
734-
)
735-
bundle_version = (
736-
task_instance.dag_version.bundle_version
737-
if task_instance.dag_version and task_instance.dag_run.bundle_version is not None
738-
else task_instance.dag_run.bundle_version
739-
)
740-
version_data = _resolve_version_data(
741-
task_instance.dag_version, task_instance.dag_run.bundle_version
742-
)
725+
from airflow.models.dag_version import _resolve_ti_callback_bundle_info
726+
727+
bundle_name, bundle_version, version_data = _resolve_ti_callback_bundle_info(task_instance)
743728
request = TaskCallbackRequest(
744729
filepath=task_instance.dag_model.relative_fileloc,
745730
ti=task_instance_to_runtime(

0 commit comments

Comments
 (0)