Skip to content

Commit b964422

Browse files
Rachel IntRachel Int
authored andcommitted
perf: route final aggregate buckets by column
1 parent f9df89e commit b964422

6 files changed

Lines changed: 290 additions & 66 deletions

File tree

‎Cargo.lock‎

Lines changed: 17 additions & 35 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,25 @@ rust-version = "1.94.0"
8383
# Define DataFusion version
8484
version = "55.1.0"
8585

86+
[patch.crates-io]
87+
arrow = { git = "https://github.com/Rachelint/arrow-rs.git", rev = "2686c17d36190e1eeebae9140d2c2efb713edde2" }
88+
arrow-arith = { git = "https://github.com/Rachelint/arrow-rs.git", rev = "2686c17d36190e1eeebae9140d2c2efb713edde2" }
89+
arrow-array = { git = "https://github.com/Rachelint/arrow-rs.git", rev = "2686c17d36190e1eeebae9140d2c2efb713edde2" }
90+
arrow-avro = { git = "https://github.com/Rachelint/arrow-rs.git", rev = "2686c17d36190e1eeebae9140d2c2efb713edde2" }
91+
arrow-buffer = { git = "https://github.com/Rachelint/arrow-rs.git", rev = "2686c17d36190e1eeebae9140d2c2efb713edde2" }
92+
arrow-cast = { git = "https://github.com/Rachelint/arrow-rs.git", rev = "2686c17d36190e1eeebae9140d2c2efb713edde2" }
93+
arrow-cmp = { git = "https://github.com/Rachelint/arrow-rs.git", rev = "2686c17d36190e1eeebae9140d2c2efb713edde2" }
94+
arrow-csv = { git = "https://github.com/Rachelint/arrow-rs.git", rev = "2686c17d36190e1eeebae9140d2c2efb713edde2" }
95+
arrow-data = { git = "https://github.com/Rachelint/arrow-rs.git", rev = "2686c17d36190e1eeebae9140d2c2efb713edde2" }
96+
arrow-flight = { git = "https://github.com/Rachelint/arrow-rs.git", rev = "2686c17d36190e1eeebae9140d2c2efb713edde2" }
97+
arrow-ipc = { git = "https://github.com/Rachelint/arrow-rs.git", rev = "2686c17d36190e1eeebae9140d2c2efb713edde2" }
98+
arrow-json = { git = "https://github.com/Rachelint/arrow-rs.git", rev = "2686c17d36190e1eeebae9140d2c2efb713edde2" }
99+
arrow-ord = { git = "https://github.com/Rachelint/arrow-rs.git", rev = "2686c17d36190e1eeebae9140d2c2efb713edde2" }
100+
arrow-row = { git = "https://github.com/Rachelint/arrow-rs.git", rev = "2686c17d36190e1eeebae9140d2c2efb713edde2" }
101+
arrow-schema = { git = "https://github.com/Rachelint/arrow-rs.git", rev = "2686c17d36190e1eeebae9140d2c2efb713edde2" }
102+
arrow-select = { git = "https://github.com/Rachelint/arrow-rs.git", rev = "2686c17d36190e1eeebae9140d2c2efb713edde2" }
103+
arrow-string = { git = "https://github.com/Rachelint/arrow-rs.git", rev = "2686c17d36190e1eeebae9140d2c2efb713edde2" }
104+
86105
[workspace.dependencies]
87106
# We turn off default-features for some dependencies here so the workspaces which inherit them can
88107
# selectively turn them on if needed, since we can override default-features = true (from false)

‎datafusion/physical-plan/src/aggregates/bucketed_aggregation.rs‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ pub(super) struct BucketedAggregation {
6666
bucket_splits: metrics::Count,
6767
/// Number of times the rows of a bucket were replaced by their aggregated state
6868
bucket_compactions: metrics::Count,
69+
/// Time spent routing rows into buckets, excluding compaction.
70+
bucket_routing_time: metrics::Time,
6971
}
7072

7173
/// What [`BucketedAggregation::output_stream`] works with while it runs.
@@ -92,6 +94,8 @@ impl BucketedAggregation {
9294
MetricBuilder::new(&agg.metrics).counter("bucket_splits", partition);
9395
let bucket_compactions =
9496
MetricBuilder::new(&agg.metrics).counter("bucket_compactions", partition);
97+
let bucket_routing_time = MetricBuilder::new(&agg.metrics)
98+
.subset_time("bucket_routing_time", partition);
9599
Self {
96100
threshold,
97101
agg,
@@ -102,9 +106,19 @@ impl BucketedAggregation {
102106
spill_manager,
103107
bucket_splits,
104108
bucket_compactions,
109+
bucket_routing_time,
105110
}
106111
}
107112

113+
pub(super) fn route(
114+
&self,
115+
buckets: &mut FinalBuckets,
116+
batch: &RecordBatch,
117+
) -> Result<()> {
118+
let _timer = self.bucket_routing_time.timer();
119+
buckets.route(batch)
120+
}
121+
108122
/// True if a schema of partial state rows can be bucketed.
109123
///
110124
/// Bucketing turns the state of a table into rows and merges those rows
@@ -167,7 +181,7 @@ impl BucketedAggregation {
167181
self.bucket_splits.add(1);
168182
buckets.expect_kept(kept);
169183
if let Some(state) = state {
170-
buckets.route(&state)?;
184+
self.route(&mut buckets, &state)?;
171185
}
172186
Ok(buckets)
173187
}
@@ -308,7 +322,7 @@ impl BucketedAggregation {
308322
let held_bytes = waiting_bytes + source_bytes;
309323

310324
if let Some(sub_buckets) = sub_buckets.as_mut() {
311-
sub_buckets.route(&batch)?;
325+
self.route(sub_buckets, &batch)?;
312326
self.compact(sub_buckets, &mut compaction_table)?;
313327
self.reserve(&output.reservation, held_bytes, sub_buckets)?;
314328
continue;

0 commit comments

Comments
 (0)