Skip to content

Commit e13736a

Browse files
committed
Merge branch 'main' into externalsorter4
2 parents d5b518c + 4e015db commit e13736a

27 files changed

Lines changed: 2548 additions & 93 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ arrow-avro = { version = "58.1.0", default-features = false, features = [
100100
"xz",
101101
] }
102102
arrow-buffer = { version = "58.1.0", default-features = false }
103+
arrow-data = { version = "58.1.0", default-features = false }
103104
arrow-flight = { version = "58.1.0", features = [
104105
"flight-sql-experimental",
105106
] }

‎datafusion/common/src/config.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,15 @@ config_namespace! {
311311
/// By default, `nulls_max` is used to follow Postgres's behavior.
312312
/// postgres rule: <https://www.postgresql.org/docs/current/queries-order.html>
313313
pub default_null_ordering: String, default = "nulls_max".to_string()
314+
315+
/// When set to true, DataFusion may remove `ORDER BY` clauses from
316+
/// subqueries or CTEs during SQL planning when their ordering cannot
317+
/// affect the result, such as when no `LIMIT` or other
318+
/// order-sensitive operator depends on them.
319+
///
320+
/// Disable this option to preserve explicit subquery ordering in the
321+
/// planned query.
322+
pub enable_subquery_sort_elimination: bool, default = true
314323
}
315324
}
316325

‎datafusion/datasource-parquet/src/opener.rs‎

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1199,10 +1199,7 @@ impl RowGroupsPrunedParquetOpen {
11991199
predicate_cache_records,
12001200
baseline_metrics: prepared.baseline_metrics,
12011201
},
1202-
|mut state| async move {
1203-
let result = state.transition().await;
1204-
result.map(|r| (r, state))
1205-
},
1202+
|state| async move { state.transition().await },
12061203
)
12071204
.fuse();
12081205

@@ -1248,15 +1245,15 @@ impl PushDecoderStreamState {
12481245
/// fetched from the [`AsyncFileReader`] and fed back into the decoder.
12491246
/// - [`Data`](DecodeResult::Data) – a decoded batch is projected and returned.
12501247
/// - [`Finished`](DecodeResult::Finished) – signals end-of-stream (`None`).
1251-
async fn transition(&mut self) -> Option<Result<RecordBatch>> {
1248+
///
1249+
/// Takes `self` by value (rather than `&mut self`) so the generated future
1250+
/// owns the state directly. This avoids a Stacked Borrows violation under
1251+
/// miri where `&mut self` creates a single opaque borrow that conflicts
1252+
/// with `unfold`'s ownership across yield points.
1253+
async fn transition(mut self) -> Option<(Result<RecordBatch>, Self)> {
12521254
loop {
12531255
match self.decoder.try_decode() {
12541256
Ok(DecodeResult::NeedsData(ranges)) => {
1255-
// IO (get_byte_ranges) and CPU (push_ranges) are still
1256-
// decoupled — they just can't live in a nested async block
1257-
// because that captures `&mut self` as one opaque borrow,
1258-
// which violates Stacked Borrows across the yield point.
1259-
// Inlining lets the compiler split the disjoint field borrows.
12601257
let data = self
12611258
.reader
12621259
.get_byte_ranges(ranges.clone())
@@ -1265,24 +1262,26 @@ impl PushDecoderStreamState {
12651262
match data {
12661263
Ok(data) => {
12671264
if let Err(e) = self.decoder.push_ranges(ranges, data) {
1268-
return Some(Err(DataFusionError::from(e)));
1265+
return Some((Err(DataFusionError::from(e)), self));
12691266
}
12701267
}
1271-
Err(e) => return Some(Err(e)),
1268+
Err(e) => return Some((Err(e), self)),
12721269
}
12731270
}
12741271
Ok(DecodeResult::Data(batch)) => {
12751272
let mut timer = self.baseline_metrics.elapsed_compute().timer();
12761273
self.copy_arrow_reader_metrics();
12771274
let result = self.project_batch(&batch);
12781275
timer.stop();
1279-
return Some(result);
1276+
// Release the borrow on baseline_metrics before moving self
1277+
drop(timer);
1278+
return Some((result, self));
12801279
}
12811280
Ok(DecodeResult::Finished) => {
12821281
return None;
12831282
}
12841283
Err(e) => {
1285-
return Some(Err(DataFusionError::from(e)));
1284+
return Some((Err(DataFusionError::from(e)), self));
12861285
}
12871286
}
12881287
}

‎datafusion/execution/src/config.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -431,6 +431,15 @@ impl SessionConfig {
431431
self
432432
}
433433

434+
/// Enables or disables elimination of `ORDER BY` clauses in subqueries
435+
/// when they are not required by order-sensitive operators.
436+
pub fn with_enable_subquery_sort_elimination(mut self, enabled: bool) -> Self {
437+
self.options_mut()
438+
.sql_parser
439+
.enable_subquery_sort_elimination = enabled;
440+
self
441+
}
442+
434443
/// Set the size of [`sort_spill_reservation_bytes`] to control
435444
/// memory pre-reservation
436445
///

‎datafusion/functions-aggregate-common/src/aggregate/count_distinct.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,5 +22,9 @@ mod native;
2222
pub use bytes::BytesDistinctCountAccumulator;
2323
pub use bytes::BytesViewDistinctCountAccumulator;
2424
pub use dict::DictionaryCountAccumulator;
25+
pub use native::Bitmap65536DistinctCountAccumulator;
26+
pub use native::Bitmap65536DistinctCountAccumulatorI16;
27+
pub use native::BoolArray256DistinctCountAccumulator;
28+
pub use native::BoolArray256DistinctCountAccumulatorI8;
2529
pub use native::FloatDistinctCountAccumulator;
2630
pub use native::PrimitiveDistinctCountAccumulator;

0 commit comments

Comments
 (0)