Skip to content

Commit 86bb0d3

Browse files
alambviiryaIsaac
authored
[branch-56] fix: reject a multi-key null-aware join without a NOT IN value key count (backport apache#27111) (apache#27176)
## Which issue does this PR close? - Backport of apache#27111 to `branch-56` - Part of apache#24461 - Closes apache#26569 Co-authored-by: Liang-Chi Hsieh <[email protected]> Co-authored-by: Isaac <[email protected]>
1 parent 6339fcb commit 86bb0d3

11 files changed

Lines changed: 328 additions & 152 deletions

File tree

‎datafusion/core/tests/sql/joins.rs‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -688,3 +688,75 @@ async fn asof_join_unparser_preserves_right_preselection() -> Result<()> {
688688
}
689689
Ok(())
690690
}
691+
692+
/// The `NOT IN` value key count of every decorrelated mark join in `sql`'s
693+
/// optimized plan, with whether the join is null-aware.
694+
async fn mark_join_value_keys(
695+
ctx: &SessionContext,
696+
sql: &str,
697+
) -> Result<Vec<(bool, usize)>> {
698+
use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion};
699+
use datafusion::logical_expr::{JoinType, LogicalPlan};
700+
701+
let plan = ctx.sql(sql).await?.into_optimized_plan()?;
702+
let mut joins = vec![];
703+
plan.apply(|plan| {
704+
if let LogicalPlan::Join(join) = plan
705+
&& join.join_type == JoinType::LeftMark
706+
{
707+
joins.push((join.null_aware, join.null_aware_value_keys));
708+
}
709+
Ok(TreeNodeRecursion::Continue)
710+
})?;
711+
Ok(joins)
712+
}
713+
714+
/// Decorrelation sets the value key count only on a null-aware join, which is
715+
/// the only one that reads it. Any other join keeps it unset (`0`), the same as a
716+
/// join built through `Join::try_new` or `LogicalPlanBuilder`, so equal plans
717+
/// compare equal however they were built.
718+
#[tokio::test]
719+
async fn decorrelated_mark_join_sets_value_keys_only_when_null_aware() -> Result<()> {
720+
let ctx = SessionContext::new();
721+
for (name, nullable) in [("t", false), ("n", true)] {
722+
let schema = Arc::new(Schema::new(vec![
723+
Field::new("a", DataType::Int32, nullable),
724+
Field::new("b", DataType::Int32, nullable),
725+
]));
726+
let batch = RecordBatch::try_new(
727+
Arc::clone(&schema),
728+
vec![
729+
Arc::new(Int32Array::from(vec![1, 2])),
730+
Arc::new(Int32Array::from(vec![3, 4])),
731+
],
732+
)?;
733+
ctx.register_table(
734+
name,
735+
Arc::new(MemTable::try_new(schema, vec![vec![batch]])?),
736+
)?;
737+
}
738+
739+
// Non-nullable keys: neither the scalar nor the multi-column `IN` is
740+
// null-aware.
741+
for sql in [
742+
"SELECT a IN (SELECT a FROM t AS s) FROM t",
743+
"SELECT (a, b) IN (SELECT a, b FROM t AS s) FROM t",
744+
] {
745+
assert_eq!(
746+
mark_join_value_keys(&ctx, sql).await?,
747+
[(false, 0)],
748+
"{sql}"
749+
);
750+
}
751+
// Nullable keys: both are null-aware and state their count.
752+
assert_eq!(
753+
mark_join_value_keys(&ctx, "SELECT a IN (SELECT a FROM n AS s) FROM n").await?,
754+
[(true, 1)]
755+
);
756+
assert_eq!(
757+
mark_join_value_keys(&ctx, "SELECT (a, b) IN (SELECT a, b FROM n AS s) FROM n")
758+
.await?,
759+
[(true, 2)]
760+
);
761+
Ok(())
762+
}

‎datafusion/expr/src/logical_plan/builder.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1440,7 +1440,7 @@ impl LogicalPlanBuilder {
14401440
schema: DFSchemaRef::new(join_schema),
14411441
null_equality,
14421442
null_aware,
1443-
null_aware_value_keys: 1,
1443+
null_aware_value_keys: 0,
14441444
})))
14451445
}
14461446

‎datafusion/expr/src/logical_plan/plan.rs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4650,6 +4650,11 @@ pub struct Join {
46504650
/// after them are correlation scope keys of a correlated subquery. A NULL
46514651
/// in a value key follows SQL three-valued logic across the whole tuple,
46524652
/// while the scope keys only select which subquery rows are compared.
4653+
///
4654+
/// `0` leaves the count unset, which is the default of [`Self::try_new`]
4655+
/// and the `LogicalPlanBuilder` join methods. Physical planning accepts an
4656+
/// unset count only for a single-key join, a scalar `NOT IN`, and rejects
4657+
/// it for more keys: no default can tell which of them are value keys.
46534658
pub null_aware_value_keys: usize,
46544659
}
46554660

@@ -4878,7 +4883,7 @@ impl Join {
48784883
schema: Arc::new(join_schema),
48794884
null_equality,
48804885
null_aware,
4881-
null_aware_value_keys: 1,
4886+
null_aware_value_keys: 0,
48824887
})
48834888
}
48844889

@@ -7228,7 +7233,7 @@ mod tests {
72287233
schema: Arc::new(left_schema.join(&right_schema)?),
72297234
null_equality: NullEquality::NullEqualsNothing,
72307235
null_aware: false,
7231-
null_aware_value_keys: 1,
7236+
null_aware_value_keys: 0,
72327237
}))
72337238
}
72347239

‎datafusion/optimizer/src/decorrelate_predicate_subquery.rs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -738,14 +738,21 @@ fn subquery_output_expr(subquery: &LogicalPlan, i: usize) -> Result<Expr> {
738738
}
739739

740740
/// Sets [`Join::null_aware_value_keys`](datafusion_expr::Join::null_aware_value_keys)
741-
/// on a join built by [`build_join`].
741+
/// on a null-aware join built by [`build_join`]. Only a null-aware join reads
742+
/// the count, so any other join keeps it unset, as [`Join::try_new`] leaves
743+
/// it, and equal joins compare equal however they were built.
744+
///
745+
/// [`Join::try_new`]: datafusion_expr::Join::try_new
742746
fn with_null_aware_value_keys(
743747
plan: LogicalPlan,
744748
null_aware_value_keys: usize,
745749
) -> Result<LogicalPlan> {
746750
let LogicalPlan::Join(join) = plan else {
747751
return internal_err!("expected a join, got {}", plan.display());
748752
};
753+
if !join.null_aware {
754+
return Ok(LogicalPlan::Join(join));
755+
}
749756
Ok(LogicalPlan::Join(
750757
join.with_null_aware_value_keys(null_aware_value_keys),
751758
))

‎datafusion/optimizer/src/eliminate_cross_join.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -345,7 +345,7 @@ fn find_inner_join(
345345
schema: join_schema,
346346
null_equality,
347347
null_aware: false,
348-
null_aware_value_keys: 1,
348+
null_aware_value_keys: 0,
349349
}));
350350
}
351351
}
@@ -369,7 +369,7 @@ fn find_inner_join(
369369
join_constraint: JoinConstraint::On,
370370
null_equality,
371371
null_aware: false,
372-
null_aware_value_keys: 1,
372+
null_aware_value_keys: 0,
373373
}))
374374
}
375375

@@ -1443,7 +1443,7 @@ mod tests {
14431443
schema: join_schema,
14441444
null_equality: NullEquality::NullEqualsNull, // Test preservation
14451445
null_aware: false,
1446-
null_aware_value_keys: 1,
1446+
null_aware_value_keys: 0,
14471447
});
14481448

14491449
// Apply filter that can create join conditions

0 commit comments

Comments
 (0)