Skip to content

Commit 1182cb9

Browse files
adriangbclaude
andcommitted
fix: count IPC padding in the recorded size of spill batches
The recorded size of a spill batch did not count padding. The IPC writer pads each buffer to the schema's alignment, and a compressed buffer is decompressed into its own allocation, rounded up to 64 bytes. With many narrow columns the padding can be larger than the data, so a merge could hold more than it reserved. SpillPieces now measures each piece as its sliced size plus the padding of each buffer, and records that size. The multi-level merge uses the same measure when it re-reads a run to check whether the run can shrink. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent d8a922f commit 1182cb9

5 files changed

Lines changed: 193 additions & 22 deletions

File tree

‎datafusion/physical-plan/src/sorts/multi_level_merge.rs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@
55
//! Create a stream that do a multi level merge stream
66
77
use crate::metrics::BaselineMetrics;
8+
use crate::spill::spill_pieces::recorded_size;
89
use crate::{EmptyRecordBatchStream, SpillManager};
910
use arrow::array::RecordBatch;
11+
use datafusion_common::config::SpillCompression;
1012
use std::fmt::{Debug, Formatter};
1113
use std::mem;
1214
use std::pin::Pin;
@@ -21,7 +23,6 @@ use crate::sorts::builder::try_grow_reservation_to_at_least;
2123
use crate::sorts::sort::get_reserved_bytes_for_record_batch_size;
2224
use crate::sorts::streaming_merge::{SortedSpillFile, StreamingMergeBuilder};
2325
use crate::spill::gc_view_arrays;
24-
use crate::spill::spill_manager::GetSlicedSize;
2526
use crate::stream::{ObservedStream, RecordBatchStreamAdapter};
2627
use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream};
2728
use datafusion_physical_expr_common::sort_expr::LexOrdering;
@@ -750,14 +751,16 @@ impl MultiLevelMergeBuilder {
750751
let mut all_singletons = true;
751752
let mut max_is_singleton = false;
752753
let mut decoded_max = 0;
754+
let compressed =
755+
self.spill_manager.compression != SpillCompression::Uncompressed;
753756
max_batch_rows = 0;
754757
while let Some(batch) = source.next().await {
755758
let batch = batch?;
756759
max_batch_rows = max_batch_rows.max(batch.num_rows());
757760
all_singletons &= batch.num_rows() == 1;
758-
decoded_max = decoded_max.max(batch.get_sliced_size()?);
761+
decoded_max = decoded_max.max(recorded_size(&batch, compressed)?);
759762
if batch.num_rows() == 1
760-
&& gc_view_arrays(&batch)?.get_sliced_size()? >= old_max
763+
&& recorded_size(&gc_view_arrays(&batch)?, compressed)? >= old_max
761764
{
762765
max_is_singleton = true;
763766
break;
@@ -1658,7 +1661,7 @@ mod tests {
16581661
)?;
16591662
let batch = source.next().await.unwrap()?;
16601663
assert_eq!(batch.num_rows(), 1);
1661-
assert!(gc_view_arrays(&batch)?.get_sliced_size()? < stored_max);
1664+
assert!(recorded_size(&gc_view_arrays(&batch)?, false)? < stored_max);
16621665
drop((source, batch));
16631666

16641667
// The stale estimate needs four batches for the minimum merge. Test

‎datafusion/physical-plan/src/sorts/sort/spill_tests.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use super::{
77
};
88
use crate::metrics::ExecutionPlanMetricsSet;
99
use crate::spill::get_record_batch_memory_size;
10-
use crate::spill::spill_manager::GetSlicedSize;
10+
use crate::spill::spill_pieces::recorded_size;
1111
use arrow::array::{
1212
ArrayRef, Decimal128Array, DictionaryArray, Int8Array, Int64Array, StringArray,
1313
StringViewArray,
@@ -1017,7 +1017,7 @@ async fn check_final_spilled_merge_releases_unused_workspace(
10171017
.iter()
10181018
.all(|spill| spill.max_record_batch_memory == large_spill_bytes)
10191019
);
1020-
let tail_spill_bytes = sort_batch(&tail, &ordering, None)?.get_sliced_size()?;
1020+
let tail_spill_bytes = recorded_size(&sort_batch(&tail, &ordering, None)?, false)?;
10211021
let single_buffer_bytes = if intermediate {
10221022
4 * large_spill_bytes // Two intermediate files remain for the final merge.
10231023
} else {

‎datafusion/physical-plan/src/spill/in_progress_spill_file.rs‎

Lines changed: 133 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
//! Define the `InProgressSpillFile` struct, which represents an in-progress spill file used for writing `RecordBatch`es to disk, created by `SpillManager`.
66
7+
use datafusion_common::config::SpillCompression;
78
use datafusion_common::{Result, internal_datafusion_err};
89
use std::sync::Arc;
910

@@ -89,7 +90,11 @@ impl InProgressSpillFile {
8990
));
9091
}
9192

92-
let pieces = SpillPieces::new(batch, self.max_batch_bytes)?;
93+
let pieces = SpillPieces::new(
94+
batch,
95+
self.max_batch_bytes,
96+
self.spill_manager.compression != SpillCompression::Uncompressed,
97+
)?;
9398

9499
if self.writer.is_none() {
95100
// Use the SpillManager's declared schema rather than the batch's schema.
@@ -147,7 +152,11 @@ impl InProgressSpillFile {
147152
));
148153
}
149154

150-
let pieces = SpillPieces::new(batch, self.max_batch_bytes)?;
155+
let pieces = SpillPieces::new(
156+
batch,
157+
self.max_batch_bytes,
158+
self.spill_manager.compression != SpillCompression::Uncompressed,
159+
)?;
151160

152161
if self.writer.is_none() {
153162
let schema = self.spill_manager.schema();
@@ -286,7 +295,7 @@ mod tests {
286295
use arrow::array::{
287296
Array, ArrayRef, AsArray, BinaryViewArray, DictionaryArray, Int32Array,
288297
Int64Array, ListBuilder, MapBuilder, StringArray, StringBuilder, StringViewArray,
289-
StructArray, UInt32Array,
298+
StructArray, UInt8Array, UInt32Array,
290299
};
291300
use arrow_schema::{DataType, Field, Fields, Schema, SchemaRef};
292301
use datafusion_common::utils::memory::get_record_batch_memory_size;
@@ -673,8 +682,8 @@ mod tests {
673682
)])),
674683
vec![Arc::new(dictionary)],
675684
)?;
676-
let pieces =
677-
SpillPieces::new(&batch, Some(1024 * 1024))?.collect::<Result<Vec<_>>>()?;
685+
let pieces = SpillPieces::new(&batch, Some(1024 * 1024), false)?
686+
.collect::<Result<Vec<_>>>()?;
678687
assert_eq!(pieces.len(), 1);
679688
assert_eq!(pieces[0].0, batch);
680689
Ok(())
@@ -733,8 +742,8 @@ mod tests {
733742
false,
734743
)]));
735744
let batch = RecordBatch::try_new(schema, vec![list_view])?;
736-
let pieces =
737-
SpillPieces::new(&batch, Some(16 * 1024))?.collect::<Result<Vec<_>>>()?;
745+
let pieces = SpillPieces::new(&batch, Some(16 * 1024), false)?
746+
.collect::<Result<Vec<_>>>()?;
738747
assert_eq!(pieces.len(), 1);
739748
assert_eq!(pieces[0].0, batch);
740749
Ok(())
@@ -760,6 +769,25 @@ mod tests {
760769
Ok(())
761770
}
762771

772+
/// With many narrow columns the IPC padding of each buffer is larger than the data, so
773+
/// the recorded size must include it or a reader decodes more than was reserved.
774+
#[tokio::test]
775+
async fn recorded_size_includes_ipc_padding() -> Result<()> {
776+
let schema = Arc::new(Schema::new(
777+
(0..4096)
778+
.map(|i| Field::new(format!("c{i}"), DataType::UInt8, false))
779+
.collect::<Vec<_>>(),
780+
));
781+
let columns = (0..4096)
782+
.map(|_| Arc::new(UInt8Array::from_iter_values(0..17)) as ArrayRef)
783+
.collect();
784+
let batch = RecordBatch::try_new(schema, columns)?;
785+
// `round_trip` checks every batch read back against the recorded size.
786+
let (read, _) = round_trip(std::slice::from_ref(&batch), 64 * 1024).await?;
787+
assert_same_rows(&read, std::slice::from_ref(&batch));
788+
Ok(())
789+
}
790+
763791
/// A piece whose dictionary keys are all null has no used values. After the unused
764792
/// values are dropped the dictionary is empty, which must not panic when measured, and
765793
/// must read back as nulls.
@@ -778,7 +806,7 @@ mod tests {
778806
// Pieces of at most 4 rows: the first ones have only null keys.
779807
let budget = 4 * WIDE + WIDE / 2;
780808
let pieces =
781-
SpillPieces::new(&batch, Some(budget))?.collect::<Result<Vec<_>>>()?;
809+
SpillPieces::new(&batch, Some(budget), false)?.collect::<Result<Vec<_>>>()?;
782810
assert!(pieces.len() > 1);
783811
assert!(
784812
pieces.iter().any(|(piece, _)| piece
@@ -788,7 +816,7 @@ mod tests {
788816
.is_empty()),
789817
"expected a piece with an empty dictionary"
790818
);
791-
SpillPieces::new(&batch.slice(0, 0), Some(budget))?
819+
SpillPieces::new(&batch.slice(0, 0), Some(budget), false)?
792820
.collect::<Result<Vec<_>>>()?;
793821

794822
let (read, _) = round_trip(std::slice::from_ref(&batch), budget).await?;
@@ -967,3 +995,99 @@ mod tests {
967995
Ok(())
968996
}
969997
}
998+
999+
#[cfg(test)]
1000+
/// The recorded size bounds what a reader decodes, for every codec and for buffer types
1001+
/// whose IPC padding or decompressed allocation is larger than their data.
1002+
mod decoded_size_bound_tests {
1003+
use super::*;
1004+
use arrow::array::*;
1005+
use arrow_schema::{DataType, Field, Schema};
1006+
use datafusion_common::config::SpillCompression;
1007+
use datafusion_common::utils::memory::get_record_batch_memory_size;
1008+
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
1009+
use datafusion_physical_expr_common::metrics::{
1010+
ExecutionPlanMetricsSet, SpillMetrics,
1011+
};
1012+
use futures::TryStreamExt;
1013+
1014+
fn cols(n: usize, rows: usize) -> Vec<(String, RecordBatch)> {
1015+
let mk = |name: &str, dt: DataType, f: &dyn Fn() -> ArrayRef| {
1016+
let schema = Arc::new(Schema::new(
1017+
(0..n)
1018+
.map(|i| Field::new(format!("c{i}"), dt.clone(), true))
1019+
.collect::<Vec<_>>(),
1020+
));
1021+
(
1022+
name.to_string(),
1023+
RecordBatch::try_new(schema, (0..n).map(|_| f()).collect()).unwrap(),
1024+
)
1025+
};
1026+
vec![
1027+
mk("u8", DataType::UInt8, &|| {
1028+
Arc::new(UInt8Array::from_iter_values((0..rows).map(|i| i as u8)))
1029+
}),
1030+
mk("i64_nulls", DataType::Int64, &|| {
1031+
Arc::new(Int64Array::from_iter(
1032+
(0..rows).map(|i| (i % 2 == 0).then_some(i as i64)),
1033+
))
1034+
}),
1035+
mk("bool", DataType::Boolean, &|| {
1036+
Arc::new(BooleanArray::from_iter((0..rows).map(|i| Some(i % 2 == 0))))
1037+
}),
1038+
mk("utf8", DataType::Utf8, &|| {
1039+
Arc::new(StringArray::from_iter_values(
1040+
(0..rows).map(|i| format!("v{i}")),
1041+
))
1042+
}),
1043+
mk("utf8view", DataType::Utf8View, &|| {
1044+
Arc::new(StringViewArray::from_iter_values(
1045+
(0..rows).map(|i| format!("a-long-string-value-{i}")),
1046+
))
1047+
}),
1048+
mk("dec256", DataType::Decimal256(40, 2), &|| {
1049+
Arc::new(
1050+
Decimal256Array::from_iter_values(
1051+
(0..rows).map(|i| arrow::datatypes::i256::from_i128(i as i128)),
1052+
)
1053+
.with_precision_and_scale(40, 2)
1054+
.unwrap(),
1055+
)
1056+
}),
1057+
]
1058+
}
1059+
1060+
#[tokio::test]
1061+
async fn recorded_size_bounds_decoded_batch() -> Result<()> {
1062+
for comp in [
1063+
SpillCompression::Uncompressed,
1064+
SpillCompression::Lz4Frame,
1065+
SpillCompression::Zstd,
1066+
] {
1067+
for (n, rows) in [(256usize, 17usize), (4, 4000)] {
1068+
for (name, batch) in cols(n, rows) {
1069+
let runtime = Arc::new(RuntimeEnvBuilder::new().build()?);
1070+
let metrics = SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0);
1071+
let m = Arc::new(
1072+
SpillManager::new(runtime, metrics, batch.schema())
1073+
.with_compression_type(comp),
1074+
);
1075+
let mut f = m
1076+
.create_in_progress_file("t")?
1077+
.with_max_batch_bytes(Some(1 << 30));
1078+
let rec = f.append_batch(&batch)?;
1079+
let file = f.finish()?.unwrap();
1080+
let read: Vec<RecordBatch> =
1081+
m.read_spill_as_stream(file, None)?.try_collect().await?;
1082+
let actual =
1083+
read.iter().map(get_record_batch_memory_size).max().unwrap();
1084+
assert!(
1085+
actual <= rec + crate::spill::SPILL_BATCH_MEMORY_MARGIN,
1086+
"{comp:?} {name} cols={n} rows={rows}: recorded {rec}, decoded {actual}"
1087+
);
1088+
}
1089+
}
1090+
}
1091+
Ok(())
1092+
}
1093+
}

‎datafusion/physical-plan/src/spill/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -710,7 +710,7 @@ fn buffer_to_bytes(buffer: Buffer) -> bytes::Bytes {
710710
}
711711

712712
// Returns the maximum byte alignment required by any field in the schema (>= 8), derived from Arrow buffer layouts.
713-
fn get_max_alignment_for_schema(schema: &Schema) -> usize {
713+
pub(crate) fn get_max_alignment_for_schema(schema: &Schema) -> usize {
714714
let minimum_alignment = 8;
715715
let mut max_alignment = minimum_alignment;
716716
for field in schema.fields() {

0 commit comments

Comments
 (0)