Skip to content

Commit b29d5b0

Browse files
committed
test: reach the shared-views gate with a computed sort key
Since runs whose keys fit keep `batch_size`, the shared-views case sorted by column keys never reached `views_share_bytes`. Sort it by a computed key, which counts as larger than the run, so the case fails without the gate.
1 parent 8eb93eb commit b29d5b0

1 file changed

Lines changed: 27 additions & 9 deletions

File tree

‎datafusion/physical-plan/src/sorts/sort/spill_tests.rs‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ use datafusion_execution::memory_pool::{
2222
GreedyMemoryPool, MemoryConsumer, MemoryLimit, MemoryPool, MemoryReservation,
2323
};
2424
use datafusion_execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder};
25-
use datafusion_physical_expr::expressions::Column;
26-
use datafusion_physical_expr::{LexOrdering, PhysicalSortExpr};
25+
use datafusion_physical_expr::expressions::{CastExpr, Column};
26+
use datafusion_physical_expr::{LexOrdering, PhysicalExpr, PhysicalSortExpr};
2727
use futures::TryStreamExt;
2828
use std::fmt::{Display, Formatter};
2929
use std::sync::{Arc, Mutex};
@@ -721,10 +721,17 @@ async fn check_merge_keys_larger_than_batches() -> Result<()> {
721721
.await?;
722722
}
723723
// Views into 1024 dictionary values share their bytes, as a
724-
// dictionary-encoded Parquet column produces. Sorting the buffered batches
725-
// before a spill grows past the pool, on `main` too.
724+
// dictionary-encoded Parquet column produces. A computed key counts as
725+
// larger than the run, so only the shared views keep these runs at
726+
// `batch_size`. Sorting the buffered batches before a spill grows past the
727+
// pool, on `main` too.
726728
let fields = [&int_fields[..], &[("c", DataType::Utf8View)]].concat();
727-
check_merge_keys(&fields, 2, 16, false, |batch| {
729+
let a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
730+
let keys = vec![
731+
Arc::new(CastExpr::new(a, DataType::Int64, None)) as Arc<dyn PhysicalExpr>,
732+
Arc::new(Column::new("b", 1)),
733+
];
734+
check_merge_exprs(&fields, keys, 16, false, |batch| {
728735
let values: StringArray = (0..1024)
729736
.map(|v| Some(format!("{batch:>4}-{v:0>59}")))
730737
.collect();
@@ -775,6 +782,20 @@ async fn check_merge_keys(
775782
buffered: usize,
776783
within_pool: bool,
777784
columns: impl Fn(i32) -> Result<Vec<ArrayRef>>,
785+
) -> Result<()> {
786+
let keys = (0..keys)
787+
.map(|i| Arc::new(Column::new(fields[i].0, i)) as Arc<dyn PhysicalExpr>)
788+
.collect();
789+
check_merge_exprs(fields, keys, buffered, within_pool, columns).await
790+
}
791+
792+
/// Like [`check_merge_keys`], sorting by `keys`.
793+
async fn check_merge_exprs(
794+
fields: &[(&str, DataType)],
795+
keys: Vec<Arc<dyn PhysicalExpr>>,
796+
buffered: usize,
797+
within_pool: bool,
798+
columns: impl Fn(i32) -> Result<Vec<ArrayRef>>,
778799
) -> Result<()> {
779800
let schema = Arc::new(Schema::new(
780801
fields
@@ -785,10 +806,7 @@ async fn check_merge_keys(
785806
let batches = (0..64)
786807
.map(|batch| Ok(RecordBatch::try_new(Arc::clone(&schema), columns(batch)?)?))
787808
.collect::<Result<Vec<_>>>()?;
788-
let ordering =
789-
LexOrdering::new((0..keys).map(|i| {
790-
PhysicalSortExpr::new_default(Arc::new(Column::new(fields[i].0, i)))
791-
}))
809+
let ordering = LexOrdering::new(keys.into_iter().map(PhysicalSortExpr::new_default))
792810
.expect("at least one sort key");
793811
let capacity =
794812
MERGE_BYTES + buffered * get_reserved_bytes_for_record_batch(&batches[0])?;

0 commit comments

Comments
 (0)