@@ -63,7 +63,9 @@ use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays};
6363use arrow:: datatypes:: { DataType , SchemaRef } ;
6464use datafusion_common:: config:: SpillCompression ;
6565use datafusion_common:: tree_node:: TreeNodeRecursion ;
66- use datafusion_common:: utils:: memory:: RecordBatchMemoryCounter ;
66+ use datafusion_common:: utils:: {
67+ get_available_parallelism, memory:: RecordBatchMemoryCounter ,
68+ } ;
6769use datafusion_common:: {
6870 DataFusionError , Result , assert_or_internal_err, internal_datafusion_err,
6971 unwrap_or_internal_err,
@@ -105,7 +107,7 @@ impl ExternalSorterMetrics {
105107/// Limits concurrent batch sorting across the partitions of one [`SortExec`].
106108#[ derive( Debug , Clone ) ]
107109struct SortConcurrency {
108- /// Each partition can use the full worker budget, but queues at most this
110+ /// Each partition can use the full concurrency budget, but queues at most this
109111 /// many jobs so it cannot put all its runs ahead of other partitions.
110112 per_partition_limit : usize ,
111113 /// Shared budget: a job holds a permit until its run has been sorted.
@@ -114,19 +116,24 @@ struct SortConcurrency {
114116
115117impl SortConcurrency {
116118 /// Returns the shared limit on multithreaded Tokio runtimes, initialized
117- /// from the current runtime's worker count on first use.
119+ /// on first use.
118120 fn for_current_runtime ( cell : & OnceLock < Self > ) -> Option < Self > {
119121 let handle = Handle :: try_current ( ) . ok ( ) ?;
120122 if handle. runtime_flavor ( ) != RuntimeFlavor :: MultiThread {
121123 return None ;
122124 }
123125 Some (
124126 cell. get_or_init ( || {
125- // Sorting is CPU-bound; more jobs would only queue for a worker.
126- let workers = handle. metrics ( ) . num_workers ( ) ;
127+ // These jobs are CPU-bound. Cap concurrency by the worker count
128+ // to avoid queuing jobs that cannot run, and by estimated CPU
129+ // parallelism to avoid oversubscribing CPUs on larger runtimes.
130+ let limit = handle
131+ . metrics ( )
132+ . num_workers ( )
133+ . min ( get_available_parallelism ( ) ) ;
127134 Self {
128- per_partition_limit : workers ,
129- permits : Arc :: new ( Semaphore :: new ( workers ) ) ,
135+ per_partition_limit : limit ,
136+ permits : Arc :: new ( Semaphore :: new ( limit ) ) ,
130137 }
131138 } )
132139 . clone ( ) ,
@@ -1289,9 +1296,10 @@ pub fn sort_batch_chunked(
12891296/// by the memory manager, by spilling to disk.
12901297///
12911298/// On multithreaded Tokio runtimes, large inputs are sorted in batches on
1292- /// separate tasks before merging. All partitions share a task limit equal to
1293- /// the worker count of the runtime that first performs this sorting. The plan
1294- /// returned by [`ExecutionPlan::reset_state`] starts with a fresh limit.
1299+ /// separate tasks before merging. All partitions share a task limit capped by
1300+ /// both the worker count of the runtime that first performs this sorting and
1301+ /// the estimated available CPU parallelism. The plan returned by
1302+ /// [`ExecutionPlan::reset_state`] starts with a fresh limit.
12951303/// Inputs sorted in a single pass, current-thread runtimes, execution outside
12961304/// Tokio, and TopK sorting continue sorting inline.
12971305#[ derive( Debug , Clone ) ]
0 commit comments