@@ -225,6 +225,16 @@ def get_callback_ti(ti):
225225 )
226226
227227
228+ def _attempt_states(ti, session):
229+ attempts = session.scalars(
230+ select(TaskInstance)
231+ .where(TaskInstance.dag_id == ti.dag_id, TaskInstance.task_id == ti.task_id)
232+ .execution_options(include_all_attempts=True)
233+ .order_by(TaskInstance.try_number)
234+ )
235+ return [(attempt.try_number, attempt.state, attempt.working_set) for attempt in attempts]
236+
237+
228238@pytest.fixture(scope="class")
229239def disable_load_example():
230240 with conf_vars({("core", "load_examples"): "false"}):
@@ -1247,6 +1257,84 @@ def test_process_executor_events_with_callback(
12471257 },
12481258 )
12491259
1260+ @pytest.mark.parametrize(
1261+ "callback_kwargs",
1262+ [
1263+ pytest.param({"on_failure_callback": lambda context: None}, id="on_failure_callback"),
1264+ pytest.param({"email": "
[email protected] "}, id="email"),
1265+ ],
1266+ )
1267+ def test_process_executor_events_callback_for_running_ti_with_retries_moves_it_to_retry(
1268+ self, callback_kwargs, dag_maker, session
1269+ ):
1270+ """Sending the callback must not close the session and detach the TI before its retry is set up."""
1271+ with dag_maker(dag_id="executor_event_running_ti_with_retries", fileloc="/test_path1/"):
1272+ EmptyOperator(task_id="task", retries=1, **callback_kwargs)
1273+ dr = dag_maker.create_dagrun()
1274+ ti = dr.task_instances[0]
1275+ executor = MockExecutor(do_update=False)
1276+ executor.callback_sink = DatabaseCallbackSink()
1277+ scheduler_job = Job()
1278+ session.add(scheduler_job)
1279+ session.flush()
1280+ self.job_runner = SchedulerJobRunner(scheduler_job, executors=[executor])
1281+ ti.state = State.RUNNING
1282+ ti.try_number = 1
1283+ ti.queued_by_job_id = scheduler_job.id
1284+ session.merge(ti)
1285+ session.commit()
1286+ executor.event_buffer[TaskInstanceUuid(ti.id)] = State.SUCCESS, None
1287+
1288+ self.job_runner._process_executor_events(executor=executor, session=session)
1289+ session.commit()
1290+
1291+ session.expire_all()
1292+ assert _attempt_states(ti, session) == [
1293+ (1, State.FAILED, None),
1294+ (2, State.UP_FOR_RETRY, True),
1295+ ]
1296+ assert session.scalar(select(func.count()).select_from(DbCallbackRequest)) == 1
1297+
1298+ def test_process_executor_events_dag_lookup_failure_keeps_session_for_rest_of_batch(
1299+ self, dag_maker, session
1300+ ):
1301+ """A failed Dag lookup must not close the session and detach the rest of the batch."""
1302+ with dag_maker(dag_id="executor_event_dag_lookup_failure"):
1303+ EmptyOperator(task_id="first", retries=1)
1304+ EmptyOperator(task_id="second", retries=1)
1305+ dr = dag_maker.create_dagrun()
1306+ executor = MockExecutor(do_update=False)
1307+ scheduler_job = Job()
1308+ session.add(scheduler_job)
1309+ session.flush()
1310+ self.job_runner = SchedulerJobRunner(scheduler_job, executors=[executor])
1311+ for ti in dr.task_instances:
1312+ ti.state = State.RUNNING
1313+ ti.try_number = 1
1314+ ti.queued_by_job_id = scheduler_job.id
1315+ session.merge(ti)
1316+ executor.event_buffer[TaskInstanceUuid(ti.id)] = State.SUCCESS, None
1317+ session.commit()
1318+ real_get_dag_for_run = self.job_runner.scheduler_dag_bag.get_dag_for_run
1319+ lookup_errors = [AirflowException("Dag lookup failed")]
1320+
1321+ def get_dag_for_run(*, dag_run, session):
1322+ if lookup_errors:
1323+ raise lookup_errors.pop()
1324+ return real_get_dag_for_run(dag_run=dag_run, session=session)
1325+
1326+ with mock.patch.object(
1327+ self.job_runner.scheduler_dag_bag, "get_dag_for_run", autospec=True, side_effect=get_dag_for_run
1328+ ):
1329+ self.job_runner._process_executor_events(executor=executor, session=session)
1330+ session.commit()
1331+
1332+ session.expire_all()
1333+ live_states = sorted(
1334+ session.scalars(select(TaskInstance.state).where(TaskInstance.dag_id == dr.dag_id))
1335+ )
1336+ assert live_states == [State.SUCCESS, State.UP_FOR_RETRY]
1337+
12501338 @mock.patch("airflow.jobs.scheduler_job_runner.TaskCallbackRequest")
12511339 def test_process_executor_events_callback_for_removed_mapped_task(
12521340 self, mock_task_callback, dag_maker, session
@@ -10632,6 +10720,33 @@ def test_heartbeat_timeout_sets_callback_type_by_retry_eligibility(
1063210720 )
1063310721 assert dispatched_callback == expected_dispatched_callback
1063410722
10723+ def test_heartbeat_timeout_with_retries_moves_ti_to_retry(self, dag_maker, session):
10724+ """Sending the heartbeat-timeout callback must not close the purge's session and detach the TI."""
10725+ with dag_maker(dag_id="hb_timeout_with_retries", session=session):
10726+ EmptyOperator(task_id="test_task", retries=1)
10727+ dag_run = dag_maker.create_dagrun(run_id="test_run", state=DagRunState.RUNNING)
10728+ executor = MockExecutor(do_update=False)
10729+ executor.callback_sink = DatabaseCallbackSink()
10730+ scheduler_job = Job()
10731+ session.add(scheduler_job)
10732+ session.flush()
10733+ self.job_runner = SchedulerJobRunner(scheduler_job, executors=[executor])
10734+ ti = dag_run.get_task_instance(task_id="test_task", session=session)
10735+ ti.state = TaskInstanceState.RUNNING
10736+ ti.try_number = 1
10737+ ti.queued_by_job_id = scheduler_job.id
10738+ ti.last_heartbeat_at = timezone.utcnow() - timedelta(seconds=600)
10739+ session.commit()
10740+
10741+ self.job_runner._find_and_purge_task_instances_without_heartbeats()
10742+
10743+ session.expire_all()
10744+ assert _attempt_states(ti, session) == [
10745+ (1, TaskInstanceState.FAILED, None),
10746+ (2, TaskInstanceState.UP_FOR_RETRY, True),
10747+ ]
10748+ assert session.scalar(select(func.count()).select_from(DbCallbackRequest)) == 1
10749+
1063510750 @pytest.mark.parametrize(
1063610751 ("retries", "callback_kind", "expected"),
1063710752 [
0 commit comments