Skip to content

Commit 93b3061

Browse files
committed
refactor: use owner prefixes for join filter metrics
1 parent c05e973 commit 93b3061

8 files changed

Lines changed: 77 additions & 69 deletions

File tree

‎docs/source/user-guide/latest/metrics.md‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -35,14 +35,14 @@ With `spark.comet.exec.join.dynamicFilter.enabled=true`, native broadcast and sh
3535
report these additional metric keys. See [Join Runtime Filters](tuning.md#join-runtime-filters) for
3636
eligibility and reader restrictions.
3737

38-
| Metric | Description |
39-
| ---------------------------------------- | ----------------------------------------------------------------- |
40-
| `dynamic_filter_rows_evaluated` | Probe rows evaluated by the runtime filter. |
41-
| `dynamic_filter_rows_pruned` | Probe rows rejected by that filter before the hash probe. |
42-
| `dynamic_filter_rows_bypassed` | Probe rows passed through while the runtime filter is inactive. |
43-
| `dynamic_filter_eval_time` | Time evaluating the runtime filter. |
44-
| `dynamic_filter_reader_filters_attached` | Executions that attach their runtime filter to a native reader. |
45-
| `dynamic_filter_reader_filters_skipped` | Executions whose probe input is ineligible for reader attachment. |
38+
| Metric | Description |
39+
| -------------------------------------- | ----------------------------------------------------------------- |
40+
| `dynamic_filter_join_rows_evaluated` | Probe rows evaluated by the runtime filter. |
41+
| `dynamic_filter_join_rows_pruned` | Probe rows rejected by that filter before the hash probe. |
42+
| `dynamic_filter_join_rows_bypassed` | Probe rows passed through while the runtime filter is inactive. |
43+
| `dynamic_filter_join_eval_time` | Time evaluating the runtime filter. |
44+
| `dynamic_filter_join_filters_attached` | Executions that attach their runtime filter to a native reader. |
45+
| `dynamic_filter_join_filters_skipped` | Executions whose probe input is ineligible for reader attachment. |
4646

4747
The row counters measure residual filtering of decoded probe batches. They exclude rows skipped
4848
by the reader. An attached filter does not guarantee that any row groups are pruned: compare the

‎native/core/src/execution/operators/dynamic_filter.rs‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -43,18 +43,21 @@ pub(crate) struct DynamicFilterExec {
4343
input: Arc<dyn ExecutionPlan>,
4444
predicate: Arc<DynamicFilterPhysicalExpr>,
4545
metrics: ExecutionPlanMetricsSet,
46+
metric_prefix: &'static str,
4647
}
4748

4849
impl DynamicFilterExec {
4950
pub(super) fn new(
5051
input: Arc<dyn ExecutionPlan>,
5152
predicate: Arc<DynamicFilterPhysicalExpr>,
5253
metrics: ExecutionPlanMetricsSet,
54+
metric_prefix: &'static str,
5355
) -> Self {
5456
Self {
5557
input,
5658
predicate,
5759
metrics,
60+
metric_prefix,
5861
}
5962
}
6063
}
@@ -116,6 +119,7 @@ impl ExecutionPlan for DynamicFilterExec {
116119
children.remove(0),
117120
Arc::clone(&self.predicate),
118121
ExecutionPlanMetricsSet::new(),
122+
self.metric_prefix,
119123
)))
120124
}
121125

@@ -131,6 +135,7 @@ impl ExecutionPlan for DynamicFilterExec {
131135
Arc::clone(&self.input),
132136
predicate,
133137
ExecutionPlanMetricsSet::new(),
138+
self.metric_prefix,
134139
)))
135140
}
136141

@@ -150,16 +155,16 @@ impl ExecutionPlan for DynamicFilterExec {
150155
let predicate = Arc::clone(&self.predicate)
151156
.with_new_children(vec![Arc::new(Column::new(key.name(), 0))])?;
152157
let input = self.input.execute(partition, context)?;
153-
let evaluated =
154-
MetricBuilder::new(&self.metrics).counter("dynamic_filter_rows_evaluated", partition);
155-
let pruned =
156-
MetricBuilder::new(&self.metrics).counter("dynamic_filter_rows_pruned", partition);
157-
let bypassed =
158-
MetricBuilder::new(&self.metrics).counter("dynamic_filter_rows_bypassed", partition);
158+
let evaluated = MetricBuilder::new(&self.metrics)
159+
.counter(format!("{}_rows_evaluated", self.metric_prefix), partition);
160+
let pruned = MetricBuilder::new(&self.metrics)
161+
.counter(format!("{}_rows_pruned", self.metric_prefix), partition);
162+
let bypassed = MetricBuilder::new(&self.metrics)
163+
.counter(format!("{}_rows_bypassed", self.metric_prefix), partition);
159164
// Only dedicated metrics: merging this helper into the Spark join must not
160165
// add its input/output counts or elapsed time to the join's existing metrics.
161-
let eval_time =
162-
MetricBuilder::new(&self.metrics).subset_time("dynamic_filter_eval_time", partition);
166+
let eval_time = MetricBuilder::new(&self.metrics)
167+
.subset_time(format!("{}_eval_time", self.metric_prefix), partition);
163168
let stream = input.map(move |batch| {
164169
let batch = batch?;
165170
let _timer = eval_time.timer();

‎native/core/src/execution/operators/dynamic_filter/tests.rs‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ async fn placeholder_updates_and_errors_are_not_hidden() {
8787
source,
8888
Arc::clone(&predicate),
8989
ExecutionPlanMetricsSet::new(),
90+
"test_filter",
9091
));
9192
let task = SessionContext::new().task_ctx();
9293
let mut stream = wrapper.execute(0, Arc::clone(&task)).unwrap();
@@ -104,9 +105,9 @@ async fn placeholder_updates_and_errors_are_not_hidden() {
104105
while let Some(batch) = stream.next().await {
105106
assert_eq!(batch.unwrap().num_rows(), 0);
106107
}
107-
assert_eq!(metric(&wrapper, "dynamic_filter_rows_bypassed"), 2);
108-
assert_eq!(metric(&wrapper, "dynamic_filter_rows_pruned"), 8);
109-
assert_eq!(metric(&wrapper, "dynamic_filter_rows_evaluated"), 8);
108+
assert_eq!(metric(&wrapper, "test_filter_rows_bypassed"), 2);
109+
assert_eq!(metric(&wrapper, "test_filter_rows_pruned"), 8);
110+
assert_eq!(metric(&wrapper, "test_filter_rows_evaluated"), 8);
110111

111112
// Reset must not preserve an old condition, even while another owner
112113
// still holds the previous predicate.
@@ -115,8 +116,8 @@ async fn placeholder_updates_and_errors_are_not_hidden() {
115116
.await
116117
.unwrap();
117118
assert_eq!(row_count(&reset_output), 10);
118-
assert_eq!(metric(&reset, "dynamic_filter_rows_pruned"), 0);
119-
assert_eq!(metric(&reset, "dynamic_filter_rows_bypassed"), 10);
119+
assert_eq!(metric(&reset, "test_filter_rows_pruned"), 0);
120+
assert_eq!(metric(&reset, "test_filter_rows_bypassed"), 10);
120121

121122
predicate.update(lit(42i32)).unwrap();
122123
let error = collect(wrapper, task).await.unwrap_err();

‎native/core/src/execution/operators/dynamic_filter_join.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ impl DynamicFilterJoinExec {
9999
reader.unwrap_or_else(|| Arc::clone(self.template.right())),
100100
Arc::clone(&predicate),
101101
self.metrics.clone(),
102+
"dynamic_filter_join",
102103
));
103104
// In particular, do not share CollectLeft's cached build future with the
104105
// template, another execution, or a reset plan.
@@ -240,9 +241,9 @@ impl ExecutionPlan for DynamicFilterJoinExec {
240241
) -> Result<SendableRecordBatchStream> {
241242
let runtime = self.build_runtime_join()?;
242243
let attachment_metric = if runtime.reader_filter_attached {
243-
"dynamic_filter_reader_filters_attached"
244+
"dynamic_filter_join_filters_attached"
244245
} else {
245-
"dynamic_filter_reader_filters_skipped"
246+
"dynamic_filter_join_filters_skipped"
246247
};
247248
MetricBuilder::new(&self.metrics)
248249
.counter(attachment_metric, partition)

‎native/core/src/execution/operators/dynamic_filter_join/tests.rs‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,7 @@ async fn completed_filter_evaluates_only_the_shared_probe_key() {
288288
probe,
289289
predicate,
290290
ExecutionPlanMetricsSet::new(),
291+
"dynamic_filter_join",
291292
));
292293
let filtered = collect(consumer, session.task_ctx()).await.unwrap();
293294
let selected = BooleanArray::from(
@@ -353,9 +354,9 @@ async fn completed_build_filters_both_sides_and_session_inlist_settings() {
353354
4,
354355
"duplicates and late build keys must match"
355356
);
356-
assert_eq!(metric(&attached, "dynamic_filter_rows_evaluated"), 202);
357-
assert!(metric(&attached, "dynamic_filter_rows_pruned") >= 199);
358-
assert_eq!(metric(&attached, "dynamic_filter_rows_bypassed"), 0);
357+
assert_eq!(metric(&attached, "dynamic_filter_join_rows_evaluated"), 202);
358+
assert!(metric(&attached, "dynamic_filter_join_rows_pruned") >= 199);
359+
assert_eq!(metric(&attached, "dynamic_filter_join_rows_bypassed"), 0);
359360
} else {
360361
assert_eq!(row_count(&actual), 0);
361362
}
@@ -559,7 +560,7 @@ async fn independent_attempts_do_not_share_build_domains() {
559560
})
560561
.collect::<Vec<_>>();
561562
assert_eq!(keys, vec![build_key]);
562-
assert_eq!(metric(&attached, "dynamic_filter_rows_pruned"), 1);
563+
assert_eq!(metric(&attached, "dynamic_filter_join_rows_pruned"), 1);
563564
}
564565
}
565566

@@ -840,7 +841,7 @@ async fn reader_filter_crosses_null_check_conjunction_and_retains_residual() {
840841
assert_eq!(
841842
plan.metrics()
842843
.unwrap()
843-
.sum_by_name("dynamic_filter_reader_filters_attached")
844+
.sum_by_name("dynamic_filter_join_filters_attached")
844845
.unwrap()
845846
.as_usize(),
846847
1
@@ -887,7 +888,7 @@ async fn reader_filter_does_not_cross_fetch_limits() {
887888
.unwrap();
888889
outputs.push(row_count(&output));
889890
if enabled {
890-
assert_eq!(metric(&plan, "dynamic_filter_reader_filters_skipped"), 1);
891+
assert_eq!(metric(&plan, "dynamic_filter_join_filters_skipped"), 1);
891892
}
892893
}
893894
assert_eq!(outputs, vec![0, 0]);
@@ -969,10 +970,10 @@ async fn reader_filter_does_not_cross_seeded_rand_probe_filter() {
969970
.and_then(|metrics| metrics.sum_by_name(name))
970971
.map_or(0, |value| value.as_usize())
971972
};
972-
assert_eq!(dynamic_metric("dynamic_filter_reader_filters_attached"), 0);
973-
assert_eq!(dynamic_metric("dynamic_filter_reader_filters_skipped"), 1);
974-
assert_eq!(dynamic_metric("dynamic_filter_rows_evaluated"), 1);
975-
assert_eq!(dynamic_metric("dynamic_filter_rows_pruned"), 0);
973+
assert_eq!(dynamic_metric("dynamic_filter_join_filters_attached"), 0);
974+
assert_eq!(dynamic_metric("dynamic_filter_join_filters_skipped"), 1);
975+
assert_eq!(dynamic_metric("dynamic_filter_join_rows_evaluated"), 1);
976+
assert_eq!(dynamic_metric("dynamic_filter_join_rows_pruned"), 0);
976977
assert_eq!(pruning_metric(&scan, "row_groups_pruned_statistics"), 0);
977978
assert_eq!(filter.metrics().unwrap().output_rows().unwrap(), 1);
978979
}
@@ -1037,7 +1038,7 @@ async fn run_parquet_join(values: Vec<i32>, enabled: bool) -> (usize, usize, usi
10371038
.unwrap();
10381039
let attached = plan
10391040
.metrics()
1040-
.and_then(|metrics| metrics.sum_by_name("dynamic_filter_reader_filters_attached"))
1041+
.and_then(|metrics| metrics.sum_by_name("dynamic_filter_join_filters_attached"))
10411042
.map_or(0, |metric| metric.as_usize());
10421043
(
10431044
row_count(&output),
@@ -1207,7 +1208,7 @@ async fn runtime_domains_release_with_streams_while_plans_remain_alive() {
12071208
assert_eq!(metrics.output_rows(), Some(1));
12081209
assert_eq!(
12091210
metrics
1210-
.sum_by_name("dynamic_filter_rows_pruned")
1211+
.sum_by_name("dynamic_filter_join_rows_pruned")
12111212
.unwrap()
12121213
.as_usize(),
12131214
1
@@ -1278,7 +1279,7 @@ async fn duplicate_heavy_builds_do_not_materialize_unreserved_inlists() {
12781279
assert_eq!(
12791280
plan.metrics()
12801281
.unwrap()
1281-
.sum_by_name("dynamic_filter_rows_pruned")
1282+
.sum_by_name("dynamic_filter_join_rows_pruned")
12821283
.unwrap()
12831284
.as_usize(),
12841285
1
@@ -1392,14 +1393,14 @@ async fn child_replacement_rechecks_join_key_types() {
13921393
.unwrap();
13931394
assert_eq!(row_count(&output), 1);
13941395
if supported {
1395-
assert_eq!(metric(&rewritten, "dynamic_filter_rows_pruned"), 1);
1396+
assert_eq!(metric(&rewritten, "dynamic_filter_join_rows_pruned"), 1);
13961397
let reset = rewritten.reset_state().unwrap();
13971398
assert!(reset.is::<DynamicFilterJoinExec>());
13981399
let reset_output = collect(Arc::clone(&reset), session.task_ctx())
13991400
.await
14001401
.unwrap();
14011402
assert_eq!(row_count(&reset_output), 1);
1402-
assert_eq!(metric(&reset, "dynamic_filter_rows_pruned"), 1);
1403+
assert_eq!(metric(&reset, "dynamic_filter_join_rows_pruned"), 1);
14031404
}
14041405
}
14051406
}

‎native/core/src/execution/planner.rs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6266,12 +6266,14 @@ mod tests {
62666266
assert_eq!(metrics.metrics["build_input_rows"], 4);
62676267
if enabled {
62686268
assert_eq!(metrics.metrics["input_rows"], 3);
6269-
assert_eq!(metrics.metrics["dynamic_filter_rows_evaluated"], 100);
6270-
assert_eq!(metrics.metrics["dynamic_filter_rows_pruned"], 97);
6271-
assert_eq!(metrics.metrics["dynamic_filter_rows_bypassed"], 0);
6269+
assert_eq!(metrics.metrics["dynamic_filter_join_rows_evaluated"], 100);
6270+
assert_eq!(metrics.metrics["dynamic_filter_join_rows_pruned"], 97);
6271+
assert_eq!(metrics.metrics["dynamic_filter_join_rows_bypassed"], 0);
62726272
} else {
62736273
assert_eq!(metrics.metrics["input_rows"], 100);
6274-
assert!(!metrics.metrics.contains_key("dynamic_filter_rows_pruned"));
6274+
assert!(!metrics
6275+
.metrics
6276+
.contains_key("dynamic_filter_join_rows_pruned"));
62756277
}
62766278
}
62776279
}

‎spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -458,17 +458,17 @@ object CometMetricNode {
458458
/** Join runtime filtering and reader attachment. These never replace join or scan metrics. */
459459
def joinDynamicFilterMetrics(sc: SparkContext): Map[String, SQLMetric] = {
460460
Map(
461-
"dynamic_filter_rows_evaluated" ->
461+
"dynamic_filter_join_rows_evaluated" ->
462462
SQLMetrics.createMetric(sc, "Probe rows evaluated by the join runtime filter"),
463-
"dynamic_filter_rows_pruned" ->
463+
"dynamic_filter_join_rows_pruned" ->
464464
SQLMetrics.createMetric(sc, "Probe rows rejected before the hash probe"),
465-
"dynamic_filter_rows_bypassed" ->
465+
"dynamic_filter_join_rows_bypassed" ->
466466
SQLMetrics.createMetric(sc, "Probe rows bypassing an inactive join runtime filter"),
467-
"dynamic_filter_eval_time" ->
467+
"dynamic_filter_join_eval_time" ->
468468
SQLMetrics.createNanoTimingMetric(sc, "Time evaluating the join runtime filter"),
469-
"dynamic_filter_reader_filters_attached" ->
469+
"dynamic_filter_join_filters_attached" ->
470470
SQLMetrics.createMetric(sc, "Join runtime filters attached to native readers"),
471-
"dynamic_filter_reader_filters_skipped" ->
471+
"dynamic_filter_join_filters_skipped" ->
472472
SQLMetrics.createMetric(sc, "Join runtime filters not eligible for native readers"))
473473
}
474474

0 commit comments

Comments
 (0)