Skip to content

Commit 4b93040

Browse files
authored
Merge branch 'main' into abstract-spill-file
2 parents ae9ecc9 + 367f08e commit 4b93040

57 files changed

Lines changed: 2800 additions & 404 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Cargo.lock‎

Lines changed: 32 additions & 23 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ hashbrown = { version = "0.17.1" }
170170
hex = { version = "0.4.3" }
171171
indexmap = "2.14.0"
172172
insta = { version = "1.47.2", features = ["glob", "filters"] }
173-
itertools = "0.14"
173+
itertools = "0.15"
174174
itoa = "1.0"
175175
liblzma = { version = "0.4.6", features = ["static"] }
176176
log = "^0.4"

‎benchmarks/queries/h2o/window.sql‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,3 +117,34 @@ SELECT id2, largest2_v2 FROM (
117117
ROW_NUMBER() OVER (PARTITION BY id2 ORDER BY v2 DESC) AS order_v2
118118
FROM large WHERE v2 IS NOT NULL
119119
) sub_query WHERE order_v2 <= 2;
120+
121+
-- Window Top-N partition cardinality sweep (id3 % N gives N distinct partitions).
122+
-- These exercise PartitionedTopKExec across cardinalities to validate it stays
123+
-- competitive with the SortExec+Filter baseline as partition count grows.
124+
-- Window Top-N: 100 partitions
125+
SELECT pk, largest2_v2 FROM (
126+
SELECT id3 % 100 AS pk, v2 AS largest2_v2,
127+
ROW_NUMBER() OVER (PARTITION BY id3 % 100 ORDER BY v2 DESC) AS order_v2
128+
FROM large WHERE v2 IS NOT NULL
129+
) sub_query WHERE order_v2 <= 2;
130+
131+
-- Window Top-N: 1,000 partitions
132+
SELECT pk, largest2_v2 FROM (
133+
SELECT id3 % 1000 AS pk, v2 AS largest2_v2,
134+
ROW_NUMBER() OVER (PARTITION BY id3 % 1000 ORDER BY v2 DESC) AS order_v2
135+
FROM large WHERE v2 IS NOT NULL
136+
) sub_query WHERE order_v2 <= 2;
137+
138+
-- Window Top-N: 10,000 partitions
139+
SELECT pk, largest2_v2 FROM (
140+
SELECT id3 % 10000 AS pk, v2 AS largest2_v2,
141+
ROW_NUMBER() OVER (PARTITION BY id3 % 10000 ORDER BY v2 DESC) AS order_v2
142+
FROM large WHERE v2 IS NOT NULL
143+
) sub_query WHERE order_v2 <= 2;
144+
145+
-- Window Top-N: 100,000 partitions
146+
SELECT pk, largest2_v2 FROM (
147+
SELECT id3 % 100000 AS pk, v2 AS largest2_v2,
148+
ROW_NUMBER() OVER (PARTITION BY id3 % 100000 ORDER BY v2 DESC) AS order_v2
149+
FROM large WHERE v2 IS NOT NULL
150+
) sub_query WHERE order_v2 <= 2;

‎datafusion/catalog-listing/src/options.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ use datafusion_datasource::file_format::FileFormat;
2323
use datafusion_expr::{Partitioning, SortExpr};
2424
use futures::StreamExt;
2525
use futures::TryStreamExt;
26+
use itertools::AllEqualValueError;
2627
use itertools::Itertools;
2728
use std::sync::Arc;
2829

@@ -404,11 +405,10 @@ impl ListingOptions {
404405

405406
match partition_keys.into_iter().all_equal_value() {
406407
Ok(v) => Ok(v),
407-
Err(None) => Ok(vec![]),
408-
Err(Some(diff)) => {
409-
let mut sorted_diff = [diff.0, diff.1];
410-
sorted_diff.sort();
411-
plan_err!("Found mixed partition values on disk {:?}", sorted_diff)
408+
Err(AllEqualValueError(None)) => Ok(vec![]),
409+
Err(AllEqualValueError(Some(mut diff))) => {
410+
diff.sort();
411+
plan_err!("Found mixed partition values on disk {:?}", diff)
412412
}
413413
}
414414
}

‎datafusion/datasource/src/mod.rs‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,22 +55,19 @@ pub mod write;
5555
pub use self::file::as_file_source;
5656
pub use self::url::ListingTableUrl;
5757
use crate::file_groups::FileGroup;
58+
use arrow::datatypes::SchemaRef;
5859
use chrono::TimeZone;
5960
use datafusion_common::stats::Precision;
6061
use datafusion_common::{ColumnStatistics, Result, TableReference};
6162
use datafusion_common::{ScalarValue, Statistics};
6263
use datafusion_physical_expr::LexOrdering;
6364
use futures::Stream;
6465
use object_store::{ObjectMeta, path::Path};
65-
pub use table_schema::{TableSchema, TableSchemaBuilder};
66-
// Remove when add_row_stats is remove
67-
use arrow::datatypes::SchemaRef;
68-
#[expect(deprecated)]
69-
pub use statistics::add_row_stats;
7066
pub use statistics::compute_all_files_statistics;
7167
use std::any::Any;
7268
use std::pin::Pin;
7369
use std::sync::Arc;
70+
pub use table_schema::{TableSchema, TableSchemaBuilder};
7471

7572
/// User-defined per-file extension data, keyed by concrete Rust type.
7673
///

‎datafusion/datasource/src/statistics.rs‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -541,14 +541,6 @@ pub fn compute_all_files_statistics(
541541
Ok((file_groups_with_stats, statistics))
542542
}
543543

544-
#[deprecated(since = "47.0.0", note = "Use Statistics::add")]
545-
pub fn add_row_stats(
546-
file_num_rows: Precision<usize>,
547-
num_rows: Precision<usize>,
548-
) -> Precision<usize> {
549-
file_num_rows.add(&num_rows)
550-
}
551-
552544
#[cfg(test)]
553545
mod tests {
554546
use super::*;

‎datafusion/ffi/src/physical_expr/metrics.rs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,9 @@ pub struct FFI_RatioMetrics {
128128
}
129129

130130
/// FFI-stable mirror of [`MetricValue`].
131+
///
132+
/// This is part of the stable ABI and must not be reordered. New variants must be
133+
/// appended at the end.
131134
#[repr(C, u8)]
132135
#[derive(Debug, Clone)]
133136
pub enum FFI_MetricValue {
@@ -170,6 +173,10 @@ pub enum FFI_MetricValue {
170173
display: SString,
171174
as_usize_value: u64,
172175
},
176+
PeakMemoryUsage {
177+
name: SString,
178+
gauge: u64,
179+
},
173180
}
174181

175182
// -----------------------------------------------------------------------------
@@ -425,6 +432,10 @@ impl From<&MetricValue> for FFI_MetricValue {
425432
name: SString::from(name.as_ref()),
426433
gauge: gauge.value() as u64,
427434
},
435+
MetricValue::PeakMemoryUsage { name, gauge } => Self::PeakMemoryUsage {
436+
name: SString::from(name.as_ref()),
437+
gauge: gauge.value() as u64,
438+
},
428439
MetricValue::Time { name, time } => Self::Time {
429440
name: SString::from(name.as_ref()),
430441
time_ns: time.value() as u64,
@@ -481,6 +492,10 @@ impl From<FFI_MetricValue> for MetricValue {
481492
name: Cow::Owned(name.into()),
482493
gauge: gauge_from_value(gauge),
483494
},
495+
FFI_MetricValue::PeakMemoryUsage { name, gauge } => Self::PeakMemoryUsage {
496+
name: Cow::Owned(name.into()),
497+
gauge: gauge_from_value(gauge),
498+
},
484499
FFI_MetricValue::Time { name, time_ns } => Self::Time {
485500
name: Cow::Owned(name.into()),
486501
time: time_from_nanos(time_ns),
@@ -624,6 +639,13 @@ mod tests {
624639
gauge,
625640
});
626641

642+
let peak_memory = Gauge::new();
643+
peak_memory.add(44);
644+
assert_value_roundtrip(MetricValue::PeakMemoryUsage {
645+
name: Cow::Borrowed("peak_mem_used"),
646+
gauge: peak_memory,
647+
});
648+
627649
let time = Time::new();
628650
time.add_duration(std::time::Duration::from_nanos(33));
629651
assert_value_roundtrip(MetricValue::Time {

0 commit comments

Comments
 (0)