Skip to content

Commit 98c2530

Browse files
viiryaIsaac
andcommitted
fix: keep a volatile NOT IN key null-aware despite a matching correlation
A volatile key is evaluated again for the join, so a correlated conjunct on the same expression proves nothing about the join key. Co-authored-by: Isaac <[email protected]>
1 parent 865e3cd commit 98c2530

1 file changed

Lines changed: 43 additions & 2 deletions

File tree

‎datafusion/optimizer/src/decorrelate_predicate_subquery.rs‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -657,6 +657,11 @@ impl InValue {
657657
/// aside. Any other conjunct is assumed to let a NULL through, which keeps
658658
/// the join null-aware.
659659
///
660+
/// A volatile key, such as one built on `random()`, is evaluated once in the
661+
/// subquery filter and once more as the join key. The two calls can disagree,
662+
/// so a conjunct on one tells nothing about the other, and such a key keeps
663+
/// the join null-aware.
664+
///
660665
/// `key_is_outer` tells on which side of the join the key is. The scope
661666
/// filters keep their outer references, and a side of a conjunct matches an
662667
/// outer key only if it names outer columns alone. A subquery column can have
@@ -671,6 +676,9 @@ fn key_may_be_null_in_scope(
671676
if !nullable {
672677
return false;
673678
}
679+
if key.is_volatile() {
680+
return true;
681+
}
674682
let key = strip_casts(key);
675683
!scope_filters
676684
.iter()
@@ -1366,8 +1374,8 @@ mod tests {
13661374
use arrow::datatypes::{DataType, Field, Schema};
13671375
use datafusion_expr::builder::table_source;
13681376
use datafusion_expr::{
1369-
TableScanBuilder, and, binary_expr, col, cube, grouping_set, out_ref_col, rollup,
1370-
table_scan,
1377+
ScalarUDF, TableScanBuilder, Volatility, and, binary_expr, col, cube,
1378+
grouping_set, out_ref_col, rollup, table_scan,
13711379
};
13721380
use datafusion_functions_aggregate::count::count_udaf;
13731381

@@ -2509,6 +2517,39 @@ mod tests {
25092517
Ok(())
25102518
}
25112519

2520+
/// A volatile key is evaluated again for the join, so a correlation on the
2521+
/// same expression says nothing about the join key, and the mark join
2522+
/// stays null-aware.
2523+
#[test]
2524+
fn mark_join_for_volatile_in_predicate_correlation_is_null_aware() -> Result<()> {
2525+
let volatile_key = |arg: Expr| {
2526+
ScalarUDF::from(
2527+
udfs::PlacementTestUDF::new().with_volatility(Volatility::Volatile),
2528+
)
2529+
.call(vec![arg])
2530+
};
2531+
let subquery = Arc::new(
2532+
LogicalPlanBuilder::from(test_table_scan_with_name("sq")?)
2533+
.filter(
2534+
col("sq.c").eq(volatile_key(out_ref_col(DataType::UInt32, "test.c"))),
2535+
)?
2536+
.project(vec![col("sq.c")])?
2537+
.build()?,
2538+
);
2539+
let plan = LogicalPlanBuilder::from(test_table_scan()?)
2540+
.filter(not_in_subquery(volatile_key(col("test.c")), subquery).is_null())?
2541+
.build()?;
2542+
2543+
let optimized = optimize_with_decorrelate(plan)?;
2544+
assert!(
2545+
has_null_aware_left_mark_join(&optimized),
2546+
"{}",
2547+
optimized.display_indent_schema()
2548+
);
2549+
2550+
Ok(())
2551+
}
2552+
25122553
/// A grouping set above the correlation puts a NULL back into the key: the
25132554
/// grand-total row of `ROLLUP` is a NULL for every outer row. The
25142555
/// correlation then no longer bounds the subquery result, and the mark

0 commit comments

Comments
 (0)