Skip to content

Commit f0ce6ef

Browse files
fix: cap sort preparation by available CPU parallelism
Use the smaller of the Tokio worker count and DataFusion's available parallelism estimate for the shared sort-preparation budget. Explain why CPU-bound jobs need both limits and update the SortExec documentation.
1 parent 702ed31 commit f0ce6ef

1 file changed

Lines changed: 18 additions & 10 deletions

File tree

  • datafusion/physical-plan/src/sorts

‎datafusion/physical-plan/src/sorts/sort.rs‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,9 @@ use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays};
6363
use arrow::datatypes::{DataType, SchemaRef};
6464
use datafusion_common::config::SpillCompression;
6565
use datafusion_common::tree_node::TreeNodeRecursion;
66-
use datafusion_common::utils::memory::RecordBatchMemoryCounter;
66+
use datafusion_common::utils::{
67+
get_available_parallelism, memory::RecordBatchMemoryCounter,
68+
};
6769
use datafusion_common::{
6870
DataFusionError, Result, assert_or_internal_err, internal_datafusion_err,
6971
unwrap_or_internal_err,
@@ -105,7 +107,7 @@ impl ExternalSorterMetrics {
105107
/// Limits concurrent batch sorting across the partitions of one [`SortExec`].
106108
#[derive(Debug, Clone)]
107109
struct SortConcurrency {
108-
/// Each partition can use the full worker budget, but queues at most this
110+
/// Each partition can use the full concurrency budget, but queues at most this
109111
/// many jobs so it cannot put all its runs ahead of other partitions.
110112
per_partition_limit: usize,
111113
/// Shared budget: a job holds a permit until its run has been sorted.
@@ -114,19 +116,24 @@ struct SortConcurrency {
114116

115117
impl SortConcurrency {
116118
/// Returns the shared limit on multithreaded Tokio runtimes, initialized
117-
/// from the current runtime's worker count on first use.
119+
/// on first use.
118120
fn for_current_runtime(cell: &OnceLock<Self>) -> Option<Self> {
119121
let handle = Handle::try_current().ok()?;
120122
if handle.runtime_flavor() != RuntimeFlavor::MultiThread {
121123
return None;
122124
}
123125
Some(
124126
cell.get_or_init(|| {
125-
// Sorting is CPU-bound; more jobs would only queue for a worker.
126-
let workers = handle.metrics().num_workers();
127+
// These jobs are CPU-bound. Cap concurrency by the worker count
128+
// to avoid queuing jobs that cannot run, and by estimated CPU
129+
// parallelism to avoid oversubscribing CPUs on larger runtimes.
130+
let limit = handle
131+
.metrics()
132+
.num_workers()
133+
.min(get_available_parallelism());
127134
Self {
128-
per_partition_limit: workers,
129-
permits: Arc::new(Semaphore::new(workers)),
135+
per_partition_limit: limit,
136+
permits: Arc::new(Semaphore::new(limit)),
130137
}
131138
})
132139
.clone(),
@@ -1289,9 +1296,10 @@ pub fn sort_batch_chunked(
12891296
/// by the memory manager, by spilling to disk.
12901297
///
12911298
/// On multithreaded Tokio runtimes, large inputs are sorted in batches on
1292-
/// separate tasks before merging. All partitions share a task limit equal to
1293-
/// the worker count of the runtime that first performs this sorting. The plan
1294-
/// returned by [`ExecutionPlan::reset_state`] starts with a fresh limit.
1299+
/// separate tasks before merging. All partitions share a task limit capped by
1300+
/// both the worker count of the runtime that first performs this sorting and
1301+
/// the estimated available CPU parallelism. The plan returned by
1302+
/// [`ExecutionPlan::reset_state`] starts with a fresh limit.
12951303
/// Inputs sorted in a single pass, current-thread runtimes, execution outside
12961304
/// Tokio, and TopK sorting continue sorting inline.
12971305
#[derive(Debug, Clone)]

0 commit comments

Comments
 (0)