Skip to content

Commit 41026ca

Browse files
viiryaIsaac
andcommitted
fix: address review feedback on null-aware value key counts
- Check the join type, partition mode and an empty `on` before the value key count, so a null-aware join no count can fix reports its real error instead of asking for `with_null_aware_value_keys`. - Decorrelation sets the count only on a null-aware join. Other joins keep it unset, as `Join::try_new` leaves it, so equal plans compare equal. - Both `with_null_aware_value_keys` setters take `Option<NonZeroUsize>`, which keeps the physical planner, the proto decoders and `eliminate_join` as plain builder chains. - Encode the count with `usize_to_wire` and decode it with a new shared `non_zero_usize_from_wire`, so both proto decoders reject a wire `0` with the same plan error. - Slim the scalar `NOT IN` test helper to the arguments that vary and flatten a leftover block in `test_null_aware_validation_value_keys`. Co-authored-by: Isaac <[email protected]>
1 parent ddbce35 commit 41026ca

12 files changed

Lines changed: 308 additions & 175 deletions

File tree

‎datafusion/common/src/utils/mod.rs‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1153,6 +1153,22 @@ where
11531153
})
11541154
}
11551155

1156+
/// Converts a wire integer to a [`NonZero<usize>`], rejecting zero and
1157+
/// out-of-range values. `context` and `field` identify the value in the
1158+
/// error message.
1159+
pub fn non_zero_usize_from_wire<T>(
1160+
value: T,
1161+
context: &str,
1162+
field: &str,
1163+
) -> Result<NonZero<usize>>
1164+
where
1165+
T: TryInto<usize> + std::fmt::Display + Copy,
1166+
{
1167+
NonZero::new(usize_from_wire(value, context, field)?).ok_or_else(|| {
1168+
_plan_datafusion_err!("{context}: {field} wire value 0 must be at least 1")
1169+
})
1170+
}
1171+
11561172
/// Converts a `usize` to a wire integer, rejecting out-of-range values.
11571173
pub fn usize_to_wire<T: TryFrom<usize>>(
11581174
value: usize,
@@ -1578,6 +1594,23 @@ mod tests {
15781594
"Error during planning: SomeExec: fetch value 256 is out of range for the plan wire format"
15791595
);
15801596

1597+
assert_eq!(
1598+
non_zero_usize_from_wire(2_u32, "SomeExec", "keys")
1599+
.unwrap()
1600+
.get(),
1601+
2
1602+
);
1603+
let err = non_zero_usize_from_wire(0_u32, "SomeExec", "keys").unwrap_err();
1604+
assert_eq!(
1605+
err.strip_backtrace(),
1606+
"Error during planning: SomeExec: keys wire value 0 must be at least 1"
1607+
);
1608+
let err = non_zero_usize_from_wire(-1_i64, "SomeExec", "keys").unwrap_err();
1609+
assert_eq!(
1610+
err.strip_backtrace(),
1611+
"Error during planning: SomeExec: keys wire value -1 is out of range for usize"
1612+
);
1613+
15811614
let max = usize_from_wire(u64::MAX, "SomeExec", "fetch");
15821615
if usize::BITS >= u64::BITS {
15831616
assert_eq!(max.unwrap(), usize::MAX);

‎datafusion/core/src/physical_planner.rs‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1892,22 +1892,22 @@ impl DefaultPhysicalPlanner {
18921892
PartitionMode::CollectLeft
18931893
};
18941894

1895-
let mut builder = HashJoinExecBuilder::new(
1896-
physical_left,
1897-
physical_right,
1898-
join_on,
1899-
*join_type,
1900-
)
1901-
.with_filter(join_filter)
1902-
.with_partition_mode(partition_mode)
1903-
.with_null_equality(*null_equality)
1904-
.with_null_aware(*null_aware);
19051895
// An unset count is passed on unset: the build accepts it
19061896
// only for a single key, a scalar `NOT IN`.
1907-
if let Some(value_keys) = null_aware_value_keys {
1908-
builder = builder.with_null_aware_value_keys(*value_keys);
1909-
}
1910-
Arc::new(builder.build()?)
1897+
Arc::new(
1898+
HashJoinExecBuilder::new(
1899+
physical_left,
1900+
physical_right,
1901+
join_on,
1902+
*join_type,
1903+
)
1904+
.with_filter(join_filter)
1905+
.with_partition_mode(partition_mode)
1906+
.with_null_equality(*null_equality)
1907+
.with_null_aware(*null_aware)
1908+
.with_null_aware_value_keys(*null_aware_value_keys)
1909+
.build()?,
1910+
)
19111911
};
19121912

19131913
// If plan was mutated previously then need to create the ExecutionPlan

‎datafusion/core/tests/physical_optimizer/join_selection.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -496,7 +496,7 @@ async fn test_null_aware_left_anti_swaps_to_right_anti(
496496
.with_filter(if with_filter { nl_join_filter() } else { None })
497497
.with_partition_mode(partition_mode)
498498
.with_null_aware(true)
499-
.with_null_aware_value_keys(NonZeroUsize::new(value_keys).unwrap())
499+
.with_null_aware_value_keys(NonZeroUsize::new(value_keys))
500500
.build()?;
501501
let original_schema = join.schema();
502502

‎datafusion/core/tests/sql/joins.rs‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -774,3 +774,75 @@ async fn multi_column_not_in_sees_null_type_values() -> Result<()> {
774774
assert_batches_eq!(["++", "++"], &filtered);
775775
Ok(())
776776
}
777+
778+
/// The `NOT IN` value key count of every decorrelated mark join in `sql`'s
779+
/// optimized plan, with whether the join is null-aware.
780+
async fn mark_join_value_keys(
781+
ctx: &SessionContext,
782+
sql: &str,
783+
) -> Result<Vec<(bool, Option<usize>)>> {
784+
use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion};
785+
use datafusion::logical_expr::{JoinType, LogicalPlan};
786+
787+
let plan = ctx.sql(sql).await?.into_optimized_plan()?;
788+
let mut joins = vec![];
789+
plan.apply(|plan| {
790+
if let LogicalPlan::Join(join) = plan
791+
&& join.join_type == JoinType::LeftMark
792+
{
793+
joins.push((join.null_aware, join.null_aware_value_keys.map(|n| n.get())));
794+
}
795+
Ok(TreeNodeRecursion::Continue)
796+
})?;
797+
Ok(joins)
798+
}
799+
800+
/// Decorrelation sets the value key count only on a null-aware join, which is
801+
/// the only one that reads it. Any other join keeps it unset, the same as a
802+
/// join built through `Join::try_new` or `LogicalPlanBuilder`, so equal plans
803+
/// compare equal however they were built.
804+
#[tokio::test]
805+
async fn decorrelated_mark_join_sets_value_keys_only_when_null_aware() -> Result<()> {
806+
let ctx = SessionContext::new();
807+
for (name, nullable) in [("t", false), ("n", true)] {
808+
let schema = Arc::new(Schema::new(vec![
809+
Field::new("a", DataType::Int32, nullable),
810+
Field::new("b", DataType::Int32, nullable),
811+
]));
812+
let batch = RecordBatch::try_new(
813+
Arc::clone(&schema),
814+
vec![
815+
Arc::new(Int32Array::from(vec![1, 2])),
816+
Arc::new(Int32Array::from(vec![3, 4])),
817+
],
818+
)?;
819+
ctx.register_table(
820+
name,
821+
Arc::new(MemTable::try_new(schema, vec![vec![batch]])?),
822+
)?;
823+
}
824+
825+
// Non-nullable keys: neither the scalar nor the multi-column `IN` is
826+
// null-aware.
827+
for sql in [
828+
"SELECT a IN (SELECT a FROM t AS s) FROM t",
829+
"SELECT (a, b) IN (SELECT a, b FROM t AS s) FROM t",
830+
] {
831+
assert_eq!(
832+
mark_join_value_keys(&ctx, sql).await?,
833+
[(false, None)],
834+
"{sql}"
835+
);
836+
}
837+
// Nullable keys: both are null-aware and state their count.
838+
assert_eq!(
839+
mark_join_value_keys(&ctx, "SELECT a IN (SELECT a FROM n AS s) FROM n").await?,
840+
[(true, Some(1))]
841+
);
842+
assert_eq!(
843+
mark_join_value_keys(&ctx, "SELECT (a, b) IN (SELECT a, b FROM n AS s) FROM n")
844+
.await?,
845+
[(true, Some(2))]
846+
);
847+
Ok(())
848+
}

‎datafusion/expr/src/logical_plan/plan.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4897,9 +4897,9 @@ impl Join {
48974897
/// [`Self::null_aware_value_keys`].
48984898
pub fn with_null_aware_value_keys(
48994899
mut self,
4900-
null_aware_value_keys: NonZeroUsize,
4900+
null_aware_value_keys: Option<NonZeroUsize>,
49014901
) -> Self {
4902-
self.null_aware_value_keys = Some(null_aware_value_keys);
4902+
self.null_aware_value_keys = null_aware_value_keys;
49034903
self
49044904
}
49054905

‎datafusion/optimizer/src/decorrelate_predicate_subquery.rs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -811,19 +811,26 @@ fn subquery_output_expr(subquery: &LogicalPlan, i: usize) -> Result<Expr> {
811811
}
812812

813813
/// Sets [`Join::null_aware_value_keys`](datafusion_expr::Join::null_aware_value_keys)
814-
/// on a join built by [`build_join`].
814+
/// on a null-aware join built by [`build_join`]. Only a null-aware join reads
815+
/// the count, so any other join keeps it unset, as [`Join::try_new`] leaves
816+
/// it, and equal joins compare equal however they were built.
817+
///
818+
/// [`Join::try_new`]: datafusion_expr::Join::try_new
815819
fn with_null_aware_value_keys(
816820
plan: LogicalPlan,
817821
null_aware_value_keys: usize,
818822
) -> Result<LogicalPlan> {
819823
let LogicalPlan::Join(join) = plan else {
820824
return internal_err!("expected a join, got {}", plan.display());
821825
};
826+
if !join.null_aware {
827+
return Ok(LogicalPlan::Join(join));
828+
}
822829
let Some(null_aware_value_keys) = NonZeroUsize::new(null_aware_value_keys) else {
823830
return internal_err!("a null-aware join needs at least one value key");
824831
};
825832
Ok(LogicalPlan::Join(
826-
join.with_null_aware_value_keys(null_aware_value_keys),
833+
join.with_null_aware_value_keys(Some(null_aware_value_keys)),
827834
))
828835
}
829836

‎datafusion/optimizer/src/eliminate_join.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -497,9 +497,8 @@ fn rewrite_join(
497497
if changed {
498498
// The join type or an input changed, so the output schema may have
499499
// narrowed; recompute it via `try_new`.
500-
Ok(Transformed::yes(LogicalPlan::Join(Join {
501-
null_aware_value_keys: join.null_aware_value_keys,
502-
..Join::try_new(
500+
Ok(Transformed::yes(LogicalPlan::Join(
501+
Join::try_new(
503502
left,
504503
right,
505504
join.on,
@@ -509,7 +508,8 @@ fn rewrite_join(
509508
join.null_equality,
510509
join.null_aware,
511510
)?
512-
})))
511+
.with_null_aware_value_keys(join.null_aware_value_keys),
512+
)))
513513
} else {
514514
// Nothing changed; reassemble the join reusing its existing schema rather
515515
// than recomputing it.

0 commit comments

Comments
 (0)