Skip to content

Commit a8d14f5

Browse files
committed
test: drop sort unit test that asserts reservation internals
sort_after_group_by_reserves_shared_output_once already covers the shared buffer reservation through a query, and each of its orderings fails on main. Also say in the doc comment of sorted_copy_keeps_buffers what goes wrong without it.
1 parent ed3b881 commit a8d14f5

1 file changed

Lines changed: 2 additions & 55 deletions

File tree

  • datafusion/physical-plan/src/sorts

‎datafusion/physical-plan/src/sorts/sort.rs‎

Lines changed: 2 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1057,7 +1057,8 @@ fn get_reserved_bytes_for_next_record_batch(
10571057
/// Sorting takes the selected rows into new buffers, except that `take` keeps
10581058
/// the data buffers of view arrays and the values of dictionaries and list
10591059
/// views. Every sorted run is charged for those buffers in full, so buffered
1060-
/// batches that share them must be charged for them each, too.
1060+
/// batches that share them must be charged for them each, too. Otherwise the
1061+
/// sorter buffers more batches than the merge of their sorted runs can hold.
10611062
fn sorted_copy_keeps_buffers(data_type: &DataType) -> bool {
10621063
match data_type {
10631064
DataType::Utf8View
@@ -3899,60 +3900,6 @@ mod tests {
38993900
Ok(())
39003901
}
39013902

3902-
/// Zero-copy slices of one batch share its buffers, so buffering them
3903-
/// reserves those buffers once. Sorting the slices as separate runs must
3904-
/// keep the shared buffers reserved while any run still holds them.
3905-
#[tokio::test]
3906-
async fn test_sliced_runs_reserve_shared_buffers_once() -> Result<()> {
3907-
let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, false)]));
3908-
let parent = RecordBatch::try_new(
3909-
Arc::clone(&schema),
3910-
vec![Arc::new(Int64Array::from_iter_values((0..4096).rev()))],
3911-
)?;
3912-
let slices: Vec<_> = (0..4).map(|i| parent.slice(i * 1024, 1024)).collect();
3913-
3914-
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(1024 * 1024));
3915-
let runtime = RuntimeEnvBuilder::new()
3916-
.with_memory_pool(Arc::clone(&pool))
3917-
.build_arc()?;
3918-
let ordering: LexOrdering =
3919-
[PhysicalSortExpr::new_default(Arc::new(Column::new("x", 0)))].into();
3920-
let mut sorter = ExternalSorter::new(
3921-
0,
3922-
Arc::clone(&schema),
3923-
ordering.clone(),
3924-
1024,
3925-
0,
3926-
0, // Sort each slice as its own run, rather than concatenating them.
3927-
SpillCompression::Uncompressed,
3928-
&ExecutionPlanMetricsSet::new(),
3929-
runtime,
3930-
)?;
3931-
for slice in &slices {
3932-
sorter.insert_batch(slice.clone()).await?;
3933-
}
3934-
let parent_bytes = get_record_batch_memory_size(&parent);
3935-
let sliced_bytes = slices
3936-
.iter()
3937-
.map(|slice| slice.get_sliced_size())
3938-
.sum::<Result<usize>>()?;
3939-
assert_eq!(pool.reserved(), parent_bytes + sliced_bytes);
3940-
3941-
let runs = std::mem::take(&mut sorter.in_mem_batches);
3942-
let mut streams = sorter.sort_run_streams(runs)?;
3943-
let last = streams.pop().unwrap();
3944-
for stream in streams {
3945-
stream.try_collect::<Vec<_>>().await?;
3946-
}
3947-
// The last run still holds the parent's buffers until it is sorted
3948-
assert_eq!(pool.reserved(), parent_bytes + slices[3].get_sliced_size()?);
3949-
3950-
let sorted = concat_batches(&schema, &last.try_collect::<Vec<_>>().await?)?;
3951-
assert_eq!(sorted, sort_batch(&slices[3], &ordering, None)?);
3952-
assert_eq!(pool.reserved(), 0);
3953-
Ok(())
3954-
}
3955-
39563903
#[test]
39573904
fn test_with_dynamic_filter() -> Result<()> {
39583905
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));

0 commit comments

Comments
 (0)