Skip to content

Commit 316dc6d

Browse files
Dispatch the highest priority tasks first in the executor (#70942)
1 parent 786dd79 commit 316dc6d

2 files changed

Lines changed: 28 additions & 2 deletions

File tree

‎airflow-core/src/airflow/executors/base_executor.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -428,7 +428,10 @@ def _emit_metrics(self, open_slots, num_running_tasks, num_queued_tasks):
428428

429429
def order_queued_tasks_by_priority(self) -> list[tuple[TaskInstanceKey, workloads.ExecuteTask]]:
430430
"""
431-
Orders the queued tasks by priority.
431+
Orders the queued tasks by priority, highest ``priority_weight`` first.
432+
433+
Consumers take workloads from the front of this list, so the highest priority
434+
tasks must come first for them to be scheduled before the rest.
432435
433436
:return: List of workloads from the queued_tasks according to the priority.
434437
"""
@@ -439,7 +442,7 @@ def order_queued_tasks_by_priority(self) -> list[tuple[TaskInstanceKey, workload
439442
return sorted(
440443
self.queued_tasks.items(),
441444
key=lambda x: x[1].ti.priority_weight,
442-
reverse=False,
445+
reverse=True,
443446
)
444447

445448
def trigger_tasks(self, open_slots: int) -> None:

‎airflow-core/tests/unit/executors/test_base_executor.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,29 @@ def test_trigger_running_tasks(dag_maker):
360360
executor._process_workloads.assert_called_once()
361361

362362

363+
@pytest.mark.db_test
364+
def test_trigger_tasks_schedules_highest_priority_first(dag_maker):
365+
"""When there are fewer open slots than queued tasks, the lowest priority ones wait."""
366+
date = timezone.utcnow()
367+
368+
with dag_maker("test_trigger_tasks_priority_order"):
369+
BaseOperator(task_id="low", priority_weight=1)
370+
BaseOperator(task_id="medium", priority_weight=5)
371+
BaseOperator(task_id="high", priority_weight=10)
372+
373+
dagrun = dag_maker.create_dagrun(logical_date=date)
374+
375+
executor = BaseExecutor()
376+
executor._process_workloads = mock.Mock(spec=lambda workloads: None)
377+
for task_instance in dagrun.task_instances:
378+
executor.queued_tasks[task_instance.key] = workloads.ExecuteTask.make(task_instance)
379+
380+
executor.trigger_tasks(open_slots=2)
381+
382+
scheduled = [workload.ti.task_id for workload in executor._process_workloads.call_args[0][0]]
383+
assert scheduled == ["high", "medium"]
384+
385+
363386
def test_debug_dump(caplog):
364387
executor = BaseExecutor()
365388
with caplog.at_level(logging.INFO):

0 commit comments

Comments
 (0)