Skip to content

Commit 69fb4e0

Browse files
refactor: keep sort reservation ownership with callers
Return per-run and shared input sizes from sort_run_sizes, and create reservations at both call sites. Preserve sizing, lazy run construction, and shared-input lifetimes.
1 parent f0ce6ef commit 69fb4e0

1 file changed

Lines changed: 15 additions & 11 deletions

File tree

  • datafusion/physical-plan/src/sorts

‎datafusion/physical-plan/src/sorts/sort.rs‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -791,7 +791,9 @@ impl ExternalSorter {
791791
concurrency: &SortConcurrency,
792792
) -> Result<Vec<SendableRecordBatchStream>> {
793793
let mut streams: Vec<_> = (0..runs.len()).map(|_| None).collect();
794-
let (sizes, shared_input) = self.sort_run_reservations(&runs)?;
794+
let (sizes, shared_input_size) = self.sort_run_sizes(&runs)?;
795+
let shared_input =
796+
shared_input_size.map(|size| Arc::new(self.reservation.split(size)));
795797
let jobs =
796798
runs.into_iter()
797799
.zip(sizes)
@@ -927,17 +929,19 @@ impl ExternalSorter {
927929
.map(|_| RecordBatchMemoryCounter::new())
928930
}
929931

930-
/// Calculates each run's reservation size and reserves any shared input.
932+
/// Calculates each run's reservation size and any shared input size.
931933
///
932934
/// Runs can share buffers, for example zero-copy slices of one larger batch,
933935
/// and `self.reservation` covers each shared buffer once. A shared buffer
934936
/// stays alive until the last run holding it is sorted, so if any buffers are
935937
/// shared, the reservation for every run's input is held until all the runs
936938
/// are sorted. Otherwise each run releases its input once it is sorted.
937-
fn sort_run_reservations(
939+
/// With shared input, each run's size covers only its sorted copy. Otherwise
940+
/// it also includes that run's input.
941+
fn sort_run_sizes(
938942
&self,
939943
runs: &[RecordBatch],
940-
) -> Result<(Vec<usize>, Option<Arc<MemoryReservation>>)> {
944+
) -> Result<(Vec<usize>, Option<usize>)> {
941945
let mut counter = self.new_buffered_batches_counter();
942946
let mut shares_buffers = false;
943947
let mut sizes = Vec::with_capacity(runs.len());
@@ -950,19 +954,17 @@ impl ExternalSorter {
950954
sizes.push((retained, run.get_sliced_size()?));
951955
}
952956

953-
let shared_input = shares_buffers.then(|| {
954-
let retained = sizes.iter().map(|(retained, _)| retained).sum();
955-
Arc::new(self.reservation.split(retained))
956-
});
957+
let shared_input_size =
958+
shares_buffers.then(|| sizes.iter().map(|(retained, _)| retained).sum());
957959

958960
let sizes = sizes
959961
.into_iter()
960-
.map(|(retained, sliced)| match shared_input {
962+
.map(|(retained, sliced)| match shared_input_size {
961963
Some(_) => sliced,
962964
None => get_reserved_bytes_for_record_batch_size(retained, sliced),
963965
})
964966
.collect();
965-
Ok((sizes, shared_input))
967+
Ok((sizes, shared_input_size))
966968
}
967969

968970
/// Sorts each run into its own stream, splitting `self.reservation` between
@@ -971,7 +973,9 @@ impl ExternalSorter {
971973
&self,
972974
runs: Vec<RecordBatch>,
973975
) -> Result<Vec<SendableRecordBatchStream>> {
974-
let (sizes, shared_input) = self.sort_run_reservations(&runs)?;
976+
let (sizes, shared_input_size) = self.sort_run_sizes(&runs)?;
977+
let shared_input =
978+
shared_input_size.map(|size| Arc::new(self.reservation.split(size)));
975979
runs.into_iter()
976980
.zip(sizes)
977981
.map(|(run, size)| {

0 commit comments

Comments
 (0)