Skip to content

Commit a44acfd

Browse files
akurmustafaalamb
authored andcommitted
Improve performance of collapse, from @akurmustafa
alamb#26 fix formatting
1 parent d2fe8a4 commit a44acfd

3 files changed

Lines changed: 14 additions & 3 deletions

File tree

‎datafusion-cli/Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/physical-expr-common/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,4 +41,5 @@ arrow = { workspace = true }
4141
datafusion-common = { workspace = true, default-features = true }
4242
datafusion-expr-common = { workspace = true }
4343
hashbrown = { workspace = true }
44+
indexmap = { workspace = true }
4445
itertools = { workspace = true }

‎datafusion/physical-expr-common/src/sort_expr.rs‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,8 @@ use arrow::datatypes::Schema;
3030
use arrow::record_batch::RecordBatch;
3131
use datafusion_common::Result;
3232
use datafusion_expr_common::columnar_value::ColumnarValue;
33-
use itertools::Itertools;
33+
use indexmap::IndexSet;
34+
use itertools::{izip, Itertools};
3435

3536
/// Represents Sort operation for a column in a RecordBatch
3637
///
@@ -568,11 +569,19 @@ impl LexRequirement {
568569
/// `vec![a Some(ASC)]`.
569570
pub fn collapse(self) -> Self {
570571
let mut output = Vec::<PhysicalSortRequirement>::new();
572+
let mut exprs = IndexSet::new();
573+
let mut reqs = vec![];
571574
for item in self {
572-
if !output.iter().any(|req| req.expr.eq(&item.expr)) {
573-
output.push(item);
575+
let PhysicalSortRequirement { expr, options: req } = item;
576+
// new insertion
577+
if exprs.insert(expr) {
578+
reqs.push(req);
574579
}
575580
}
581+
debug_assert_eq!(reqs.len(), exprs.len());
582+
for (expr, req) in izip!(exprs, reqs) {
583+
output.push(PhysicalSortRequirement::new(expr, req));
584+
}
576585
LexRequirement::new(output)
577586
}
578587
}

0 commit comments

Comments
 (0)