Skip to content

Commit 548a662

Browse files
committed
Simplify semi/anti sort-merge summaries and cover adverse cases
1 parent 38eccc9 commit 548a662

15 files changed

Lines changed: 1536 additions & 1906 deletions

File tree

‎datafusion/common/src/config.rs‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -924,14 +924,6 @@ config_namespace! {
924924
/// Support for build_side.num_rows() >= u32::MAX will be added in the future.
925925
pub perfect_hash_join_min_key_density: f64, default = 0.15
926926

927-
/// Summarize eligible residual predicates in semi/anti sort-merge joins.
928-
/// Uses one representative or an extremum per equality-key group instead
929-
/// of repeatedly searching the group's rows. Supports inequality and
930-
/// ordered comparisons, side-local filters, and OR combinations. Other
931-
/// predicates retain the general join implementation. This option does
932-
/// not change join selection or apply to hash joins.
933-
pub enable_sort_merge_join_existence_summary: bool, default = false
934-
935927
/// When set to true, record batches will be examined between each operator and
936928
/// small batches will be coalesced into larger batches. This is helpful when there
937929
/// are highly selective filters or joins that could produce tiny output batches. The

‎datafusion/core/tests/sql/joins.rs‎

Lines changed: 0 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -27,92 +27,6 @@ use datafusion_sql::unparser::plan_to_sql;
2727

2828
use super::*;
2929

30-
#[tokio::test]
31-
async fn sql_sort_merge_join_existence_summary_uses_execution_setting() -> Result<()> {
32-
use datafusion::physical_plan::joins::SortMergeJoinExec;
33-
34-
let ctx = SessionContext::new();
35-
// More than one target partition is required for the SQL planner to choose
36-
// sort-merge joins, even though these small inputs may remain unpartitioned.
37-
for setting in [
38-
"SET datafusion.optimizer.prefer_hash_join = false",
39-
"SET datafusion.execution.target_partitions = 2",
40-
] {
41-
ctx.sql(setting).await?.collect().await?;
42-
}
43-
ctx.sql(
44-
"CREATE TABLE documents(id BIGINT, document_id BIGINT, version BIGINT) AS VALUES
45-
(1, 10, 1), (2, 10, 2), (3, 10, 2),
46-
(4, 20, 1), (5, 20, NULL), (6, 30, NULL)",
47-
)
48-
.await?
49-
.collect()
50-
.await?;
51-
52-
// Exercise default-off, SQL opt-in, and SQL opt-out. Each iteration builds
53-
// the real physical plan rather than constructing a join directly.
54-
for setting in [None, Some(true), Some(false)] {
55-
let enabled = setting.unwrap_or(false);
56-
if let Some(setting) = setting {
57-
ctx.sql(&format!(
58-
"SET datafusion.execution.enable_sort_merge_join_existence_summary = {setting}"
59-
))
60-
.await?
61-
.collect()
62-
.await?;
63-
}
64-
let plan = ctx
65-
.sql(
66-
"SELECT current.id FROM documents current WHERE NOT EXISTS (
67-
SELECT 1 FROM documents other
68-
WHERE other.document_id = current.document_id
69-
AND other.version > current.version
70-
)",
71-
)
72-
.await?
73-
.create_physical_plan()
74-
.await?;
75-
let batches = collect(Arc::clone(&plan), ctx.task_ctx()).await?;
76-
assert_batches_sorted_eq!(
77-
[
78-
"+----+", "| id |", "+----+", "| 2 |", "| 3 |", "| 4 |", "| 5 |",
79-
"| 6 |", "+----+",
80-
],
81-
&batches
82-
);
83-
84-
let mut pending = vec![plan];
85-
let mut joins = 0;
86-
let mut activated = 0;
87-
let mut summarized_rows = 0;
88-
while let Some(node) = pending.pop() {
89-
if node.is::<SortMergeJoinExec>() {
90-
joins += 1;
91-
let metrics = node.metrics().unwrap();
92-
for metric in metrics.iter() {
93-
match metric.value().name() {
94-
"existence_summary_enabled" => {
95-
activated += metric.value().as_usize();
96-
}
97-
"existence_summary_inner_rows" => {
98-
summarized_rows += metric.value().as_usize();
99-
}
100-
"existence_summary_fallback" => {
101-
assert_eq!(metric.value().as_usize(), 0);
102-
}
103-
_ => {}
104-
}
105-
}
106-
}
107-
pending.extend(node.children().into_iter().cloned());
108-
}
109-
assert!(joins > 0, "SQL did not select a sort-merge join");
110-
assert_eq!(activated > 0, enabled);
111-
assert_eq!(summarized_rows > 0, enabled);
112-
}
113-
Ok(())
114-
}
115-
11630
#[tokio::test]
11731
async fn volatile_join_filter_preserves_evaluations_below_min() -> Result<()> {
11832
use arrow::array::record_batch;

0 commit comments

Comments
 (0)