Skip to content

Commit f5ec361

Browse files
Fix cleared tasks getting stuck when a Dag run has no version (#71696)
A Dag run can have no version of its own: runs carried over from Airflow 2 predate the version columns, and `airflow db clean` can remove the version a run was created with. Clearing such a run left its tasks without a version too, and a task instance with no version is never enqueued — so the task sat in queued until it timed out, while its revived run held up new ones. The clear dialogs offered no way out either, since they only showed the run-on-latest option when there was a version to compare against.
1 parent 0cb0081 commit f5ec361

11 files changed

Lines changed: 344 additions & 38 deletions

File tree

‎airflow-core/docs/administration-and-deployment/dag-bundles.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,10 @@ The setting is resolved using the following precedence (highest to lowest):
276276
4. **Per-call-site fallback**: ``False`` for clear/rerun, ``True`` for backfills (preserving
277277
the historical default for each path)
278278

279+
One exception: a Dag run with no version of its own — carried over from Airflow 2, or its version
280+
since removed by ``airflow db clean`` — has nothing to preserve, so clearing it always uses the
281+
latest version and bundle version regardless of the resolved setting.
282+
279283
Global Configuration
280284
~~~~~~~~~~~~~~~~~~~~
281285

‎airflow-core/src/airflow/models/dagbag.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,10 @@ def clear_cache(self) -> int:
209209

210210
@staticmethod
211211
def _version_from_dag_run(dag_run: DagRun, *, session: Session) -> UUID | None:
212-
if not dag_run.bundle_version:
212+
# A run with no version of its own can only resolve to the latest. Runs carried over from
213+
# Airflow 2 are like this, as are runs whose version `airflow db clean` has since deleted --
214+
# the latter keep their bundle version, so they would otherwise resolve to nothing at all.
215+
if not dag_run.bundle_version or not dag_run.created_dag_version_id:
213216
if dag_version := DagVersion.get_latest_version(dag_id=dag_run.dag_id, session=session):
214217
return dag_version.id
215218

‎airflow-core/src/airflow/models/taskinstance.py‎

Lines changed: 45 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -352,6 +352,27 @@ def _update_dagrun_to_latest_version(
352352
session.flush()
353353

354354

355+
def _pin_versionless_tis_to_run_version(dag_run: DagRun, dag_version_id: UUID, session: Session) -> None:
356+
"""
357+
Give the run's unfinished task instances a dag version if they have none.
358+
359+
Once the run is pinned the scheduler stops backfilling versions onto them, and one
360+
without a version is never enqueued.
361+
"""
362+
session.execute(
363+
update(TaskInstance)
364+
.where(
365+
TaskInstance.dag_id == dag_run.dag_id,
366+
TaskInstance.run_id == dag_run.run_id,
367+
TaskInstance.dag_version_id.is_(None),
368+
# State.unfinished holds None, which SQL IN never matches.
369+
or_(TaskInstance.state.is_(None), TaskInstance.state.in_(State.unfinished)),
370+
)
371+
.values(dag_version_id=dag_version_id)
372+
.execution_options(synchronize_session="evaluate")
373+
)
374+
375+
355376
def clear_task_instances(
356377
tis: list[TaskInstance],
357378
session: Session,
@@ -372,7 +393,9 @@ def clear_task_instances(
372393
:param session: current session
373394
:param dag_run_state: state to set finished DagRuns to.
374395
If set to False, DagRuns state will not be changed.
375-
:param run_on_latest_version: whether to run on latest serialized DAG and Bundle version
396+
:param run_on_latest_version: whether to run on latest serialized DAG and Bundle version.
397+
A run with no version of its own uses the latest either way, since there is nothing
398+
else for it to run on; a task instance with no version joins its run's.
376399
377400
:meta private:
378401
"""
@@ -396,7 +419,10 @@ def clear_task_instances(
396419
# the task is terminated and becomes eligible for retry.
397420
else:
398421
dr = ti.dag_run
399-
if run_on_latest_version:
422+
# A run with no version of its own has nothing to re-run on but the latest, and the
423+
# run loop below moves it there.
424+
use_latest_version = run_on_latest_version or dr.created_dag_version_id is None
425+
if use_latest_version:
400426
ti_dag = scheduler_dagbag.get_latest_version_of_dag(ti.dag_id, session=session)
401427
else:
402428
ti_dag = scheduler_dagbag.get_dag_for_run(dag_run=dr, session=session)
@@ -418,11 +444,15 @@ def clear_task_instances(
418444
ti.state = None
419445
ti.external_executor_id = None
420446
ti.clear_next_method_args()
421-
# Match DagVersion to latest serialized DAG when run_on_latest_version.
422-
if run_on_latest_version:
447+
# Match DagVersion to latest serialized DAG when running on the latest version.
448+
if use_latest_version:
423449
latest_dag_version = DagVersion.get_latest_version(ti.dag_id, session=session)
424450
if latest_dag_version is not None:
425451
ti.dag_version_id = latest_dag_version.id
452+
elif ti.dag_version_id is None:
453+
# One without a version is never enqueued, and the run keeps its own, so it can
454+
# only go there.
455+
ti.dag_version_id = dr.created_dag_version_id
426456
session.merge(ti)
427457

428458
if dag_run_state is not False and tis:
@@ -461,10 +491,14 @@ def clear_task_instances(
461491

462492
_recalculate_dagrun_queued_at_deadlines(dr, dr.queued_at, session)
463493

494+
# A run with no version of its own has nothing to preserve, so the latest is all
495+
# it can be re-run on. Runs migrated from Airflow 2 are like this, as are runs
496+
# whose version `airflow db clean` has since deleted.
497+
use_latest_version = run_on_latest_version or dr.created_dag_version_id is None
464498
if dr.state in State.finished_dr_states:
465499
dr.state = dag_run_state
466500
dr.start_date = timezone.utcnow()
467-
if run_on_latest_version:
501+
if use_latest_version:
468502
dr_dag = scheduler_dagbag.get_latest_version_of_dag(dr.dag_id, session=session)
469503
dag_version = DagVersion.get_latest_version(dr.dag_id, session=session)
470504
if dag_version:
@@ -478,14 +512,14 @@ def clear_task_instances(
478512
dr_dag = scheduler_dagbag.get_dag_for_run(dag_run=dr, session=session)
479513
if not dr_dag:
480514
log.warning("No serialized dag found for dag '%s'", dr.dag_id)
481-
if dr_dag and not dr_dag.disable_bundle_versioning and run_on_latest_version:
515+
if dr_dag and not dr_dag.disable_bundle_versioning and use_latest_version:
482516
bundle_version = dr.dag_model.bundle_version
483-
if bundle_version is not None and run_on_latest_version:
517+
if bundle_version is not None:
484518
dr.bundle_version = bundle_version
485519
if dag_run_state == DagRunState.QUEUED:
486520
dr.last_scheduling_decision = None
487521
dr.start_date = None
488-
elif run_on_latest_version:
522+
elif use_latest_version:
489523
# Queued/running DagRun: update DR to latest version/bundle for workloads that use it.
490524
dag_version = DagVersion.get_latest_version(dr.dag_id, session=session)
491525
if dag_version and dr.created_dag_version_id != dag_version.id:
@@ -499,6 +533,9 @@ def clear_task_instances(
499533
bundle_version = dr.dag_model.bundle_version
500534
if bundle_version is not None:
501535
dr.bundle_version = bundle_version
536+
537+
if dr.created_dag_version_id:
538+
_pin_versionless_tis_to_run_version(dr, dr.created_dag_version_id, session)
502539
for ti in tis:
503540
ti.context_carrier = new_task_run_carrier(ti.dag_run.context_carrier)
504541
session.flush()

‎airflow-core/src/airflow/ui/public/i18n/locales/en/dags.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@
7777
"preventRunningTasks": "Prevent rerun if task is running",
7878
"queueNew": "Queue up new tasks",
7979
"runOnLatestVersion": "Run with latest bundle version",
80+
"runOnLatestVersionForced": "Always uses the latest — there's no earlier version to go back to",
8081
"upstream": "Upstream"
8182
}
8283
},

‎airflow-core/src/airflow/ui/src/components/Clear/Run/ClearRunDialog.tsx‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,17 @@ const ClearRunDialog = ({ dagRun, onClose, open }: Props) => {
6565
dagId,
6666
});
6767

68+
// Offered only where it changes the outcome. A non-versioned bundle (e.g. LocalDagBundle)
69+
// leaves bundle_version null and resolves to the latest serialized Dag at run time anyway,
70+
// so unless the run has no version at all the option would be a no-op there.
71+
const { runOnLatestVersionForced, shouldShowRunOnLatestOption } = getRunOnLatestVersionState({
72+
latestBundleVersion: dagDetails?.bundle_version,
73+
latestDagVersionNumber: dagDetails?.latest_dag_version?.version_number,
74+
selectedBundleVersion: dagRun.bundle_version,
75+
selectedDagVersionNumber: dagRun.dag_versions.at(-1)?.version_number,
76+
selectedVersionMissing: dagRun.dag_versions.length === 0,
77+
});
78+
6879
const { setValue: setRunOnLatestVersion, value: runOnLatestVersion } = useRerunWithLatestVersion({
6980
dagLevelConfig: dagDetails?.rerun_with_latest_version,
7081
});
@@ -94,17 +105,6 @@ const ClearRunDialog = ({ dagRun, onClose, open }: Props) => {
94105
onSuccessConfirm: handleClose,
95106
});
96107

97-
// Non-versioned bundles (e.g. LocalDagBundle) always leave bundle_version null and
98-
// resolve to the latest serialized Dag at run time, so "run on latest" is a no-op there.
99-
// Offer it only when re-running on the latest would actually change the outcome:
100-
// the run's Dag version differs from the latest while the bundle is versioned
101-
// (latest bundle_version present), or the run's bundle version differs from the latest.
102-
const { shouldShowRunOnLatestOption } = getRunOnLatestVersionState({
103-
latestBundleVersion: dagDetails?.bundle_version,
104-
latestDagVersionNumber: dagDetails?.latest_dag_version?.version_number,
105-
selectedBundleVersion: dagRun.bundle_version,
106-
selectedDagVersionNumber: dagRun.dag_versions.at(-1)?.version_number,
107-
});
108108
const shouldShowBundleVersionOption = shouldShowRunOnLatestOption && !onlyNew;
109109

110110
return (
@@ -160,8 +160,14 @@ const ClearRunDialog = ({ dagRun, onClose, open }: Props) => {
160160
>
161161
{shouldShowBundleVersionOption ? (
162162
<Checkbox
163-
checked={runOnLatestVersion}
163+
checked={runOnLatestVersionForced || runOnLatestVersion}
164+
disabled={runOnLatestVersionForced}
164165
onCheckedChange={(event) => setRunOnLatestVersion(Boolean(event.checked))}
166+
title={
167+
runOnLatestVersionForced
168+
? translate("dags:runAndTaskActions.options.runOnLatestVersionForced")
169+
: undefined
170+
}
165171
>
166172
{translate("dags:runAndTaskActions.options.runOnLatestVersion")}
167173
</Checkbox>

‎airflow-core/src/airflow/ui/src/components/Clear/TaskInstance/ClearGroupTaskInstanceDialog.tsx‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,11 @@ import { useTranslation } from "react-i18next";
2222
import { CgRedo } from "react-icons/cg";
2323
import { useParams } from "react-router-dom";
2424

25-
import { useDagServiceGetDagDetails, useTaskInstanceServiceGetTaskInstances } from "openapi/queries";
25+
import {
26+
useDagRunServiceGetDagRun,
27+
useDagServiceGetDagDetails,
28+
useTaskInstanceServiceGetTaskInstances,
29+
} from "openapi/queries";
2630
import type { LightGridTaskInstanceSummary, TaskInstanceResponse } from "openapi/requests/types.gen";
2731
import { ActionAccordion } from "src/components/ActionAccordion";
2832
import { useRerunWithLatestVersion } from "src/components/Clear/useRerunWithLatestVersion";
@@ -80,14 +84,20 @@ export const ClearGroupTaskInstanceDialog = ({ onClose, open, taskInstance }: Pr
8084

8185
const groupTaskIds = groupTaskInstances?.task_instances.map((ti) => ti.task_id) ?? [];
8286

83-
const { dagVersionsDiffer, shouldShowRunOnLatestOption } = getRunOnLatestVersionState({
84-
latestBundleVersion: dagDetails?.bundle_version,
85-
latestDagVersionNumber: dagDetails?.latest_dag_version?.version_number,
86-
selectedDagVersionNumber: taskInstance.dag_version_number,
87-
// Fall back to legacy heuristic when grid summary has no version (older API).
88-
useLatestBundleVersionAsFallback: true,
87+
const { data: dagRun } = useDagRunServiceGetDagRun({ dagId, dagRunId: runId }, undefined, {
88+
enabled: open,
8989
});
9090

91+
const { dagVersionsDiffer, runOnLatestVersionForced, shouldShowRunOnLatestOption } =
92+
getRunOnLatestVersionState({
93+
latestBundleVersion: dagDetails?.bundle_version,
94+
latestDagVersionNumber: dagDetails?.latest_dag_version?.version_number,
95+
selectedDagVersionNumber: taskInstance.dag_version_number,
96+
selectedVersionMissing: dagRun?.dag_versions.length === 0,
97+
// Fall back to legacy heuristic when grid summary has no version (older API).
98+
useLatestBundleVersionAsFallback: true,
99+
});
100+
91101
// dagVersionsDiffer becomes the fallback so the historical "auto-check when versions
92102
// differ" heuristic still applies when neither DAG-level nor global config is set.
93103
const { setValue: setRunOnLatestVersion, value: runOnLatestVersion } = useRerunWithLatestVersion({
@@ -182,8 +192,14 @@ export const ClearGroupTaskInstanceDialog = ({ onClose, open, taskInstance }: Pr
182192
>
183193
{shouldShowRunOnLatestOption ? (
184194
<Checkbox
185-
checked={runOnLatestVersion}
195+
checked={runOnLatestVersionForced || runOnLatestVersion}
196+
disabled={runOnLatestVersionForced}
186197
onCheckedChange={(event) => setRunOnLatestVersion(Boolean(event.checked))}
198+
title={
199+
runOnLatestVersionForced
200+
? translate("dags:runAndTaskActions.options.runOnLatestVersionForced")
201+
: undefined
202+
}
187203
>
188204
{translate("dags:runAndTaskActions.options.runOnLatestVersion")}
189205
</Checkbox>

‎airflow-core/src/airflow/ui/src/components/Clear/TaskInstance/ClearTaskInstanceDialog.tsx‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import { useEffect, useMemo, useState } from "react";
2121
import { useTranslation } from "react-i18next";
2222
import { CgRedo } from "react-icons/cg";
2323

24-
import { useDagServiceGetDagDetails } from "openapi/queries";
24+
import { useDagRunServiceGetDagRun, useDagServiceGetDagDetails } from "openapi/queries";
2525
import type { ClearTaskInstancesBody, TaskInstanceResponse } from "openapi/requests/types.gen";
2626
import { ActionAccordion } from "src/components/ActionAccordion";
2727
import { taskInstanceKey } from "src/components/ActionAccordion/columns";
@@ -110,13 +110,19 @@ const ClearTaskInstanceDialog = (props: Props) => {
110110
dagId,
111111
});
112112

113-
const { dagVersionsDiffer, shouldShowRunOnLatestOption } = getRunOnLatestVersionState({
114-
latestBundleVersion: dagDetails?.bundle_version,
115-
latestDagVersionNumber: dagDetails?.latest_dag_version?.version_number,
116-
selectedBundleVersion: taskInstance?.dag_version?.bundle_version,
117-
selectedDagVersionNumber: taskInstance?.dag_version?.version_number,
113+
const { data: dagRun } = useDagRunServiceGetDagRun({ dagId, dagRunId }, undefined, {
114+
enabled: openDialog,
118115
});
119116

117+
const { dagVersionsDiffer, runOnLatestVersionForced, shouldShowRunOnLatestOption } =
118+
getRunOnLatestVersionState({
119+
latestBundleVersion: dagDetails?.bundle_version,
120+
latestDagVersionNumber: dagDetails?.latest_dag_version?.version_number,
121+
selectedBundleVersion: taskInstance?.dag_version?.bundle_version,
122+
selectedDagVersionNumber: taskInstance?.dag_version?.version_number,
123+
selectedVersionMissing: dagRun?.dag_versions.length === 0,
124+
});
125+
120126
// dagVersionsDiffer becomes the fallback so the historical "auto-check when versions
121127
// differ" heuristic still applies when neither DAG-level nor global config is set.
122128
const { setValue: setRunOnLatestVersion, value: runOnLatestVersion } = useRerunWithLatestVersion({
@@ -264,8 +270,14 @@ const ClearTaskInstanceDialog = (props: Props) => {
264270
>
265271
{shouldShowRunOnLatestOption ? (
266272
<Checkbox
267-
checked={runOnLatestVersion}
273+
checked={runOnLatestVersionForced || runOnLatestVersion}
274+
disabled={runOnLatestVersionForced}
268275
onCheckedChange={(event) => setRunOnLatestVersion(Boolean(event.checked))}
276+
title={
277+
runOnLatestVersionForced
278+
? translate("dags:runAndTaskActions.options.runOnLatestVersionForced")
279+
: undefined
280+
}
269281
>
270282
{translate("dags:runAndTaskActions.options.runOnLatestVersion")}
271283
</Checkbox>

‎airflow-core/src/airflow/ui/src/components/Clear/TaskInstance/runOnLatestVersion.test.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,15 +137,27 @@ describe("getRunOnLatestVersionState", () => {
137137
name: "does not show for group fallback when latest bundle is missing",
138138
useLatestBundleVersionAsFallback: true,
139139
},
140+
{
141+
expectedDagVersionsDiffer: false,
142+
expectedRunOnLatestVersionForced: true,
143+
expectedShouldShowRunOnLatestOption: true,
144+
// A null latest bundle version pins the case that matters: the option is forced even
145+
// on a non-versioned bundle, where it would otherwise never be offered.
146+
latestBundleVersion: null,
147+
name: "forces and shows the option when the selection has no Dag version at all",
148+
selectedVersionMissing: true,
149+
},
140150
])(
141151
"$name",
142152
({
143153
expectedDagVersionsDiffer,
154+
expectedRunOnLatestVersionForced = false,
144155
expectedShouldShowRunOnLatestOption,
145156
latestBundleVersion,
146157
latestDagVersionNumber,
147158
selectedBundleVersion,
148159
selectedDagVersionNumber,
160+
selectedVersionMissing,
149161
useLatestBundleVersionAsFallback,
150162
}) => {
151163
expect(
@@ -154,10 +166,12 @@ describe("getRunOnLatestVersionState", () => {
154166
latestDagVersionNumber,
155167
selectedBundleVersion,
156168
selectedDagVersionNumber,
169+
selectedVersionMissing,
157170
useLatestBundleVersionAsFallback,
158171
}),
159172
).toEqual({
160173
dagVersionsDiffer: expectedDagVersionsDiffer,
174+
runOnLatestVersionForced: expectedRunOnLatestVersionForced,
161175
shouldShowRunOnLatestOption: expectedShouldShowRunOnLatestOption,
162176
});
163177
},

‎airflow-core/src/airflow/ui/src/components/Clear/TaskInstance/runOnLatestVersion.ts‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,26 @@ type RunOnLatestVersionParams = {
2222
readonly latestDagVersionNumber?: number | null;
2323
readonly selectedBundleVersion?: string | null;
2424
readonly selectedDagVersionNumber?: number | null;
25+
/**
26+
* True when the *run* being cleared has no Dag version at all, which is the case for
27+
* anything carried over from Airflow 2. There is nothing to re-run it on but the latest
28+
* version, so the backend forces that regardless of the request. Keep this keyed off the
29+
* run: a task instance with no version of its own is given its run's version, not the
30+
* latest, so deriving this from the task instance would promise the wrong thing.
31+
*/
32+
readonly selectedVersionMissing?: boolean;
2533
readonly useLatestBundleVersionAsFallback?: boolean;
2634
};
2735

2836
type RunOnLatestVersionState = {
2937
readonly dagVersionsDiffer: boolean;
38+
/**
39+
* Drives how the checkbox renders, not what is submitted. A clear can span several runs
40+
* (via past/future) while the request carries one flag for all of them, so forcing it
41+
* would pin runs the user never selected. The backend forces each version-less run on
42+
* its own instead.
43+
*/
44+
readonly runOnLatestVersionForced: boolean;
3045
readonly shouldShowRunOnLatestOption: boolean;
3146
};
3247

@@ -38,6 +53,7 @@ export const getRunOnLatestVersionState = ({
3853
latestDagVersionNumber,
3954
selectedBundleVersion,
4055
selectedDagVersionNumber,
56+
selectedVersionMissing = false,
4157
useLatestBundleVersionAsFallback = false,
4258
}: RunOnLatestVersionParams): RunOnLatestVersionState => {
4359
const dagVersionsDiffer =
@@ -55,7 +71,10 @@ export const getRunOnLatestVersionState = ({
5571

5672
return {
5773
dagVersionsDiffer,
74+
runOnLatestVersionForced: selectedVersionMissing,
5875
shouldShowRunOnLatestOption:
59-
(dagVersionsDiffer && hasBundleVersion(latestBundleVersion)) || shouldShowForBundleVersion,
76+
selectedVersionMissing ||
77+
(dagVersionsDiffer && hasBundleVersion(latestBundleVersion)) ||
78+
shouldShowForBundleVersion,
6079
};
6180
};

0 commit comments

Comments
 (0)