Skip to content

Commit d0a9842

Browse files
authored
Add skewness metric to nodes that execute in partitioned mode (apache#25977) (#188)
<!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> Related to apache#23237 and [feat(metric): Add output skewness metric to detect skewed plans easier](apache#21211) - Closes #. I think it would be useful to add the skewness metric for nodes that execute in partitioned mdoe -- right now the only node that includes that metric in the explain analyze is DataSourceExec for parquet. This is specially useful to check how many partitions were idle during Aggregations or HashJoins for example. The PR adds a new metric `output_rows_skew` in the Explain Analyze output for the following nodes: - RepartitionExec - HashJoinExec - AggregateExec - BoundedWindowAggExec - WindowAggExec Note that for a plan that has a Parquet DataSource the skewness might not be the same as for the rest of the nodes, for example we might have the following scenario: ``` DataSourceExec [100, 100, 100, 100] → 0% skewness, data is even FilterExec [100, 0, 0, 0] → 100% (filter only matches in partition 0) AggregateExec [ 3, 0, 0, 0] → 100% (Partial: groups, not rows) RepartitionExec Hash [1, 1, 1, 0] → ~11% (3 groups re-spread by key) ``` I added a Explain Analyze test Yes, a new metric will show in the explain analyze plan for the nodes mentioned above. (cherry picked from commit c2baac5)
1 parent a358de8 commit d0a9842

8 files changed

Lines changed: 100 additions & 6 deletions

File tree

‎datafusion/core/tests/sql/explain_analyze.rs‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ use super::*;
1919
use insta::assert_snapshot;
2020
use rstest::rstest;
2121

22+
use datafusion::catalog::MemTable;
2223
use datafusion::config::ConfigOptions;
2324
use datafusion::physical_plan::display::DisplayableExecutionPlan;
2425
use datafusion::physical_plan::metrics::Timestamp;
@@ -907,6 +908,83 @@ async fn parquet_explain_analyze() {
907908
);
908909
}
909910

911+
/// Operators that run once per partition report `output_rows_skew`. A single
912+
/// join/group/window key hashes every row to one partition, so each
913+
/// hash-partitioned operator must report the maximum skew.
914+
#[tokio::test]
915+
#[cfg_attr(coverage, ignore)]
916+
async fn explain_analyze_output_rows_skew_on_hash_partitioned_operators() {
917+
let mut config = SessionConfig::new().with_target_partitions(4);
918+
// Force a partitioned (hash repartitioned) join instead of `CollectLeft`
919+
let options = config.options_mut();
920+
options.optimizer.hash_join_single_partition_threshold = 0;
921+
options.optimizer.hash_join_single_partition_threshold_rows = 0;
922+
let ctx = SessionContext::new_with_config(config);
923+
// Spread the rows over 4 input partitions so they get hash repartitioned
924+
// (a single input partition already satisfies the hash distribution)
925+
let schema = Arc::new(Schema::new(vec![
926+
Field::new("k", DataType::Int64, false),
927+
Field::new("v", DataType::Int64, false),
928+
]));
929+
let partitions = (0..4)
930+
.map(|p| {
931+
let v = (p * 25..(p + 1) * 25).collect::<Vec<i64>>();
932+
let batch = RecordBatch::try_new(
933+
Arc::clone(&schema),
934+
vec![
935+
Arc::new(Int64Array::from(vec![1; v.len()])),
936+
Arc::new(Int64Array::from(v)),
937+
],
938+
)
939+
.unwrap();
940+
vec![batch]
941+
})
942+
.collect();
943+
let table = MemTable::try_new(schema, partitions).unwrap();
944+
ctx.register_table("t", Arc::new(table)).unwrap();
945+
946+
let sql = "EXPLAIN ANALYZE \
947+
SELECT a.k, count(*) FROM t a JOIN t b ON a.k = b.k GROUP BY a.k";
948+
let formatted = explain_analyze_output(&ctx, sql).await;
949+
for operator in [
950+
"RepartitionExec: partitioning=Hash",
951+
"HashJoinExec: mode=Partitioned",
952+
"AggregateExec: mode=SinglePartitioned",
953+
] {
954+
assert_operator_metric(&formatted, operator, "output_rows_skew=100%");
955+
}
956+
957+
let sql = "EXPLAIN ANALYZE \
958+
SELECT k, \
959+
row_number() OVER (PARTITION BY k ORDER BY v), \
960+
sum(v) OVER (PARTITION BY k) \
961+
FROM t";
962+
let formatted = explain_analyze_output(&ctx, sql).await;
963+
for operator in ["BoundedWindowAggExec", "WindowAggExec"] {
964+
assert_operator_metric(&formatted, operator, "output_rows_skew=100%");
965+
}
966+
}
967+
968+
async fn explain_analyze_output(ctx: &SessionContext, sql: &str) -> String {
969+
let actual = execute_to_batches(ctx, sql).await;
970+
arrow::util::pretty::pretty_format_batches(&actual)
971+
.unwrap()
972+
.to_string()
973+
}
974+
975+
fn assert_operator_metric(formatted: &str, operator: &str, metric: &str) {
976+
// Leading space so `WindowAggExec` does not match `BoundedWindowAggExec`
977+
let needle = format!(" {operator}");
978+
let line = formatted
979+
.lines()
980+
.find(|line| line.contains(&needle))
981+
.unwrap_or_else(|| panic!("{operator} not found in plan:\n{formatted}"));
982+
assert!(
983+
line.contains(metric),
984+
"expected {metric} on {operator}, got:\n{line}"
985+
);
986+
}
987+
910988
// This test reproduces the behavior described in
911989
// https://github.com/apache/datafusion/issues/16684 where projection
912990
// pushdown with recursive CTEs could fail to remove unused columns

‎datafusion/physical-expr-common/src/metrics/mod.rs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,22 @@ impl MetricsSet {
229229
self.metrics.push(metric)
230230
}
231231

232+
/// Returns this set with the derived `output_rows_skew` metric appended,
233+
/// or unchanged if no partition reported `output_rows` yet.
234+
///
235+
/// This is typically used in `ExecutionPlan::metrics` of operators that
236+
/// execute in multiple partitions (e.g. repartitions, partitioned joins and
237+
/// aggregations), where it shows how evenly their output rows are spread
238+
/// across partitions. With a single partition the skew is always `0%`.
239+
///
240+
/// See [`BaselineMetrics::output_rows_skew_metric`] for how skew is computed.
241+
pub fn with_output_rows_skew(mut self) -> Self {
242+
if let Some(output_rows_skew) = BaselineMetrics::output_rows_skew_metric(&self) {
243+
self.push(output_rows_skew);
244+
}
245+
self
246+
}
247+
232248
/// Returns an iterator across all metrics
233249
pub fn iter(&self) -> impl Iterator<Item = &Arc<Metric>> {
234250
self.metrics.iter()

‎datafusion/physical-plan/src/aggregates/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2128,7 +2128,7 @@ impl ExecutionPlan for AggregateExec {
21282128
}
21292129

21302130
fn metrics(&self) -> Option<MetricsSet> {
2131-
Some(self.metrics.clone_inner())
2131+
Some(self.metrics.clone_inner().with_output_rows_skew())
21322132
}
21332133

21342134
fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {

‎datafusion/physical-plan/src/joins/hash_join/exec.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1580,7 +1580,7 @@ impl ExecutionPlan for HashJoinExec {
15801580
}
15811581

15821582
fn metrics(&self) -> Option<MetricsSet> {
1583-
Some(self.metrics.clone_inner())
1583+
Some(self.metrics.clone_inner().with_output_rows_skew())
15841584
}
15851585

15861586
fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {

‎datafusion/physical-plan/src/repartition/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1768,7 +1768,7 @@ impl ExecutionPlan for RepartitionExec {
17681768
}
17691769

17701770
fn metrics(&self) -> Option<MetricsSet> {
1771-
Some(self.metrics.clone_inner())
1771+
Some(self.metrics.clone_inner().with_output_rows_skew())
17721772
}
17731773

17741774
fn child_stats_requests(&self, _partition: Option<usize>) -> Vec<ChildStats> {

‎datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -528,7 +528,7 @@ impl ExecutionPlan for BoundedWindowAggExec {
528528
}
529529

530530
fn metrics(&self) -> Option<MetricsSet> {
531-
Some(self.metrics.clone_inner())
531+
Some(self.metrics.clone_inner().with_output_rows_skew())
532532
}
533533

534534
fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {

‎datafusion/physical-plan/src/windows/window_agg_exec.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -320,7 +320,7 @@ impl ExecutionPlan for WindowAggExec {
320320
}
321321

322322
fn metrics(&self) -> Option<MetricsSet> {
323-
Some(self.metrics.clone_inner())
323+
Some(self.metrics.clone_inner().with_output_rows_skew())
324324
}
325325

326326
fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {

‎datafusion/sqllogictest/test_files/joins.slt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5185,7 +5185,7 @@ LEFT ANTI JOIN (
51855185
) t2 ON t1.k = t2.k;
51865186
----
51875187
Plan with Metrics
5188-
01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(k@0, k@0)], metrics=[output_rows=2, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=1, build_mem_used=<slt:ignore>, array_map_created_count=0, build_input_batches=0, build_input_rows=0, input_batches=1, input_rows=2, build_time=<slt:ignore>, join_time=<slt:ignore>, avg_fanout=N/A (0/0), probe_hit_rate=0% (0/2)]
5188+
01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(k@0, k@0)], metrics=[output_rows=2, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=1, build_mem_used=<slt:ignore>, array_map_created_count=0, build_input_batches=0, build_input_rows=0, input_batches=1, input_rows=2, build_time=<slt:ignore>, join_time=<slt:ignore>, avg_fanout=N/A (0/0), output_rows_skew=0%, probe_hit_rate=0% (0/2)]
51895189
02)--ProjectionExec: expr=[column1@0 as k], metrics=[output_rows=0, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=0, expr_0_eval_time=<slt:ignore>]
51905190
03)----FilterExec: column1@0 != 1, metrics=[output_rows=0, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=0, selectivity=0% (0/1)]
51915191
04)------DataSourceExec: partitions=1, partition_sizes=[1], metrics=[]

0 commit comments

Comments
 (0)