Skip to content

Commit c2baac5

Browse files
authored
Add skewness metric to nodes that execute in partitioned mode (apache#25977)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> Related to apache#23237 and [feat(metric): Add output skewness metric to detect skewed plans easier](apache#21211) - Closes #. ## Rationale for this change I think it would be useful to add the skewness metric for nodes that execute in partitioned mdoe -- right now the only node that includes that metric in the explain analyze is DataSourceExec for parquet. This is specially useful to check how many partitions were idle during Aggregations or HashJoins for example. ## What changes are included in this PR? The PR adds a new metric `output_rows_skew` in the Explain Analyze output for the following nodes: - RepartitionExec - HashJoinExec - AggregateExec - BoundedWindowAggExec - WindowAggExec Note that for a plan that has a Parquet DataSource the skewness might not be the same as for the rest of the nodes, for example we might have the following scenario: ``` DataSourceExec [100, 100, 100, 100] → 0% skewness, data is even FilterExec [100, 0, 0, 0] → 100% (filter only matches in partition 0) AggregateExec [ 3, 0, 0, 0] → 100% (Partial: groups, not rows) RepartitionExec Hash [1, 1, 1, 0] → ~11% (3 groups re-spread by key) ``` ## What is the testing strategy for this PR? I added a Explain Analyze test ## Are there any user-facing changes? Yes, a new metric will show in the explain analyze plan for the nodes mentioned above.
1 parent 62f2943 commit c2baac5

9 files changed

Lines changed: 101 additions & 7 deletions

File tree

‎datafusion/core/tests/sql/explain_analyze.rs‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ use super::*;
1919
use insta::assert_snapshot;
2020
use rstest::rstest;
2121

22+
use datafusion::catalog::MemTable;
2223
use datafusion::config::ConfigOptions;
2324
use datafusion::physical_plan::display::DisplayableExecutionPlan;
2425
use datafusion::physical_plan::metrics::Timestamp;
@@ -908,6 +909,83 @@ async fn parquet_explain_analyze() {
908909
);
909910
}
910911

912+
/// Operators that run once per partition report `output_rows_skew`. A single
913+
/// join/group/window key hashes every row to one partition, so each
914+
/// hash-partitioned operator must report the maximum skew.
915+
#[tokio::test]
916+
#[cfg_attr(coverage, ignore)]
917+
async fn explain_analyze_output_rows_skew_on_hash_partitioned_operators() {
918+
let mut config = SessionConfig::new().with_target_partitions(4);
919+
// Force a partitioned (hash repartitioned) join instead of `CollectLeft`
920+
let options = config.options_mut();
921+
options.optimizer.hash_join_single_partition_threshold = 0;
922+
options.optimizer.hash_join_single_partition_threshold_rows = 0;
923+
let ctx = SessionContext::new_with_config(config);
924+
// Spread the rows over 4 input partitions so they get hash repartitioned
925+
// (a single input partition already satisfies the hash distribution)
926+
let schema = Arc::new(Schema::new(vec![
927+
Field::new("k", DataType::Int64, false),
928+
Field::new("v", DataType::Int64, false),
929+
]));
930+
let partitions = (0..4)
931+
.map(|p| {
932+
let v = (p * 25..(p + 1) * 25).collect::<Vec<i64>>();
933+
let batch = RecordBatch::try_new(
934+
Arc::clone(&schema),
935+
vec![
936+
Arc::new(Int64Array::from(vec![1; v.len()])),
937+
Arc::new(Int64Array::from(v)),
938+
],
939+
)
940+
.unwrap();
941+
vec![batch]
942+
})
943+
.collect();
944+
let table = MemTable::try_new(schema, partitions).unwrap();
945+
ctx.register_table("t", Arc::new(table)).unwrap();
946+
947+
let sql = "EXPLAIN ANALYZE \
948+
SELECT a.k, count(*) FROM t a JOIN t b ON a.k = b.k GROUP BY a.k";
949+
let formatted = explain_analyze_output(&ctx, sql).await;
950+
for operator in [
951+
"RepartitionExec: partitioning=Hash",
952+
"HashJoinExec: mode=Partitioned",
953+
"AggregateExec: mode=SinglePartitioned",
954+
] {
955+
assert_operator_metric(&formatted, operator, "output_rows_skew=100%");
956+
}
957+
958+
let sql = "EXPLAIN ANALYZE \
959+
SELECT k, \
960+
row_number() OVER (PARTITION BY k ORDER BY v), \
961+
sum(v) OVER (PARTITION BY k) \
962+
FROM t";
963+
let formatted = explain_analyze_output(&ctx, sql).await;
964+
for operator in ["BoundedWindowAggExec", "WindowAggExec"] {
965+
assert_operator_metric(&formatted, operator, "output_rows_skew=100%");
966+
}
967+
}
968+
969+
async fn explain_analyze_output(ctx: &SessionContext, sql: &str) -> String {
970+
let actual = execute_to_batches(ctx, sql).await;
971+
arrow::util::pretty::pretty_format_batches(&actual)
972+
.unwrap()
973+
.to_string()
974+
}
975+
976+
fn assert_operator_metric(formatted: &str, operator: &str, metric: &str) {
977+
// Leading space so `WindowAggExec` does not match `BoundedWindowAggExec`
978+
let needle = format!(" {operator}");
979+
let line = formatted
980+
.lines()
981+
.find(|line| line.contains(&needle))
982+
.unwrap_or_else(|| panic!("{operator} not found in plan:\n{formatted}"));
983+
assert!(
984+
line.contains(metric),
985+
"expected {metric} on {operator}, got:\n{line}"
986+
);
987+
}
988+
911989
/// The Parquet scan's `elapsed_compute` must cover decoding, which dominates a
912990
/// full scan, and not only the projection of batches that are already decoded.
913991
/// See <https://github.com/apache/datafusion/issues/18195>.

‎datafusion/physical-expr-common/src/metrics/mod.rs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,22 @@ impl MetricsSet {
250250
self.metrics.push(metric)
251251
}
252252

253+
/// Returns this set with the derived `output_rows_skew` metric appended,
254+
/// or unchanged if no partition reported `output_rows` yet.
255+
///
256+
/// This is typically used in `ExecutionPlan::metrics` of operators that
257+
/// execute in multiple partitions (e.g. repartitions, partitioned joins and
258+
/// aggregations), where it shows how evenly their output rows are spread
259+
/// across partitions. With a single partition the skew is always `0%`.
260+
///
261+
/// See [`BaselineMetrics::output_rows_skew_metric`] for how skew is computed.
262+
pub fn with_output_rows_skew(mut self) -> Self {
263+
if let Some(output_rows_skew) = BaselineMetrics::output_rows_skew_metric(&self) {
264+
self.push(output_rows_skew);
265+
}
266+
self
267+
}
268+
253269
/// Return the metrics whose partition ID equals `partition`.
254270
///
255271
/// Unpartitioned metrics are excluded; an unknown partition yields an empty

‎datafusion/physical-plan/src/aggregates/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2600,7 +2600,7 @@ impl ExecutionPlan for AggregateExec {
26002600
}
26012601

26022602
fn metrics(&self) -> Option<MetricsSet> {
2603-
Some(self.metrics.clone_inner())
2603+
Some(self.metrics.clone_inner().with_output_rows_skew())
26042604
}
26052605

26062606
fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {

‎datafusion/physical-plan/src/joins/hash_join/exec.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1863,7 +1863,7 @@ impl ExecutionPlan for HashJoinExec {
18631863
}
18641864

18651865
fn metrics(&self) -> Option<MetricsSet> {
1866-
Some(self.metrics.clone_inner())
1866+
Some(self.metrics.clone_inner().with_output_rows_skew())
18671867
}
18681868

18691869
fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {

‎datafusion/physical-plan/src/repartition/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1884,7 +1884,7 @@ impl ExecutionPlan for RepartitionExec {
18841884
}
18851885

18861886
fn metrics(&self) -> Option<MetricsSet> {
1887-
Some(self.metrics.clone_inner())
1887+
Some(self.metrics.clone_inner().with_output_rows_skew())
18881888
}
18891889

18901890
fn child_stats_requests(&self, _partition: Option<usize>) -> Vec<ChildStats> {

‎datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -528,7 +528,7 @@ impl ExecutionPlan for BoundedWindowAggExec {
528528
}
529529

530530
fn metrics(&self) -> Option<MetricsSet> {
531-
Some(self.metrics.clone_inner())
531+
Some(self.metrics.clone_inner().with_output_rows_skew())
532532
}
533533

534534
fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {

‎datafusion/physical-plan/src/windows/window_agg_exec.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -320,7 +320,7 @@ impl ExecutionPlan for WindowAggExec {
320320
}
321321

322322
fn metrics(&self) -> Option<MetricsSet> {
323-
Some(self.metrics.clone_inner())
323+
Some(self.metrics.clone_inner().with_output_rows_skew())
324324
}
325325

326326
fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {

‎datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -338,7 +338,7 @@ query TT
338338
explain analyze select rgsel.b from dim join rgsel on dim.column1 = rgsel.b;
339339
----
340340
Plan with Metrics
341-
01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(column1@0, b@0)], projection=[b@1], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=1, build_mem_used=<slt:ignore>, array_map_created_count=0, build_input_batches=1, build_input_rows=3, input_batches=2, input_rows=3, build_time=<slt:ignore>, join_time=<slt:ignore>, avg_fanout=100% (3/3), probe_hit_rate=100% (3/3)]
341+
01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(column1@0, b@0)], projection=[b@1], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=1, build_mem_used=<slt:ignore>, array_map_created_count=0, build_input_batches=1, build_input_rows=3, input_batches=2, input_rows=3, build_time=<slt:ignore>, join_time=<slt:ignore>, avg_fanout=100% (3/3), output_rows_skew=0%, probe_hit_rate=100% (3/3)]
342342
02)--DataSourceExec: partitions=1, partition_sizes=[1], metrics=[]
343343
03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_row_group_pruning/rgsel.parquet]]}, projection=[b], file_type=parquet, predicate=DynamicFilter [ b@1 >= 1000 AND b@1 <= 3050 AND hash_lookup ], dynamic_rg_pruning=eligible, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 >= 1000 AND b_null_count@1 != row_count@2 AND b_min@3 <= 3050 AND b_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(b_min@3, b_max@0, 3/4096 buckets), required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=2, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 2 matched, row_groups_pruned_bloom_filter=2 total → 2 matched, page_index_pages_pruned=20 total → 3 matched, page_index_rows_pruned=200 total → 30 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, bytes_processed=<slt:ignore>, bytes_scanned=<slt:ignore>, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=27, row_groups_pruned_dynamic_filter=0, predicate_cache_inner_records=200, predicate_cache_records=13, bloom_filter_eval_time=<slt:ignore>, metadata_load_time=<slt:ignore>, page_index_eval_time=<slt:ignore>, row_pushdown_eval_time=<slt:ignore>, statistics_eval_time=<slt:ignore>, time_elapsed_opening=<slt:ignore>, time_elapsed_processing=<slt:ignore>, time_elapsed_scanning_total=<slt:ignore>, time_elapsed_scanning_until_data=<slt:ignore>, output_rows_skew=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]
344344

‎datafusion/sqllogictest/test_files/joins.slt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5263,7 +5263,7 @@ LEFT ANTI JOIN (
52635263
) t2 ON t1.k = t2.k;
52645264
----
52655265
Plan with Metrics
5266-
01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(k@0, k@0)], metrics=[output_rows=2, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=1, build_mem_used=<slt:ignore>, array_map_created_count=0, build_input_batches=0, build_input_rows=0, input_batches=1, input_rows=2, build_time=<slt:ignore>, join_time=<slt:ignore>, avg_fanout=N/A (0/0), probe_hit_rate=0% (0/2)]
5266+
01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(k@0, k@0)], metrics=[output_rows=2, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=1, build_mem_used=<slt:ignore>, array_map_created_count=0, build_input_batches=0, build_input_rows=0, input_batches=1, input_rows=2, build_time=<slt:ignore>, join_time=<slt:ignore>, avg_fanout=N/A (0/0), output_rows_skew=0%, probe_hit_rate=0% (0/2)]
52675267
02)--ProjectionExec: expr=[column1@0 as k], metrics=[output_rows=0, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=0, expr_0_eval_time=<slt:ignore>]
52685268
03)----FilterExec: column1@0 != 1, metrics=[output_rows=0, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=0, selectivity=0% (0/1)]
52695269
04)------DataSourceExec: partitions=1, partition_sizes=[1], metrics=[]

0 commit comments

Comments
 (0)