Skip to content

Commit 0eb4fda

Browse files
viiryaIsaac
andcommitted
test: pin the scope-key plans in slt, drop tests the fuzz covers
- EXPLAINs in null_aware_anti_join.slt for the NULL-rejecting filter that makes `b = y` a scope key (`mc_scope`), and for a cast over a dictionary on either side, which replace the substring checks of `multi_column_not_in_keeps_cast_dictionary_as_value_key`. - Update the Test 19 header for the scope-key layout. - Merge the scope map fallback test into the lazy-build test, which ran the same baseline. - Drop the third `roundtrip_join_null_aware` case and the slt result queries that the null-aware fuzz already checks against NOT EXISTS. Co-authored-by: Isaac <[email protected]>
1 parent 043950b commit 0eb4fda

5 files changed

Lines changed: 95 additions & 109 deletions

File tree

‎datafusion/core/tests/sql/joins.rs‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -787,32 +787,3 @@ async fn multi_column_not_in_sees_null_type_values() -> Result<()> {
787787
assert_batches_eq!(["++", "++"], &filtered);
788788
Ok(())
789789
}
790-
791-
/// A cast to a common type makes the hidden NULL of a dictionary a real NULL
792-
/// under a type that hides none, and the cast keeps the non-nullable field.
793-
/// The element still stays a value key, on either side of the `IN`.
794-
#[tokio::test]
795-
async fn multi_column_not_in_keeps_cast_dictionary_as_value_key() -> Result<()> {
796-
let with_null = || -> Result<ArrayRef> {
797-
dictionary(vec![0, 1], Arc::new(Int32Array::from(vec![None, Some(3)])))
798-
};
799-
for (b, y) in [
800-
// `CAST(o.b AS Int64)` on the outer side.
801-
(
802-
with_null()?,
803-
Arc::new(Int64Array::from(vec![2])) as ArrayRef,
804-
),
805-
// `CAST(i.y AS Int64)` in the subquery.
806-
(
807-
Arc::new(Int64Array::from(vec![2, 3])) as ArrayRef,
808-
dictionary(vec![1], Arc::new(Int32Array::from(vec![None, Some(2)])))?,
809-
),
810-
] {
811-
let ctx = hidden_null_context(b, y)?;
812-
let plan = ctx.sql(HIDDEN_NULL_FILTERED).await?.into_optimized_plan()?;
813-
let plan = plan.display_indent().to_string();
814-
assert!(plan.contains("CAST("), "{plan}");
815-
assert!(plan.contains("null_aware(value_keys=2)"), "{plan}");
816-
}
817-
Ok(())
818-
}

‎datafusion/physical-plan/src/joins/hash_join/exec.rs‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10634,7 +10634,9 @@ mod tests {
1063410634

1063510635
/// The hash table over the build rows' correlation scope keys is only
1063610636
/// looked up for probe rows with a NULL value key, so a probe side
10637-
/// without one never builds it, and the result is the same either way.
10637+
/// without one never builds it. A memory pool that holds the build side
10638+
/// but not that table makes the join pair the NULL-valued probe rows with
10639+
/// every build row instead. The result is the same in all three runs.
1063810640
#[tokio::test]
1063910641
async fn null_aware_scope_map_built_only_for_null_probe_values() -> Result<()> {
1064010642
// `(0, 0)` matches the build row `(0, 0)`; `(1, 1)` matches nothing.
@@ -10650,21 +10652,15 @@ mod tests {
1065010652
with_null >= without_null + 10_000 * size_of::<u32>(),
1065110653
"{without_null} -> {with_null}"
1065210654
);
10653-
Ok(())
10654-
}
10655-
10656-
/// A memory pool that holds the build side but not the scope map: the
10657-
/// join pairs the NULL-valued probe rows with every build row instead,
10658-
/// and gives the same result.
10659-
#[tokio::test]
10660-
async fn null_aware_scope_map_falls_back_when_memory_is_short() -> Result<()> {
10661-
let (_, without_null) =
10662-
run_scope_key_anti_join(None, vec![Some(0), Some(1)], None).await?;
10655+
// Not enough memory for the scope map.
1066310656
let limit = without_null + 10_000 * size_of::<u32>() / 2;
1066410657
let (rows, build_mem_used) =
1066510658
run_scope_key_anti_join(None, vec![Some(0), None], Some(limit)).await?;
10666-
assert_eq!(rows, 9_899);
10667-
assert_eq!(build_mem_used, without_null);
10659+
assert_eq!(rows, 9_899, "fallback without the scope map");
10660+
assert_eq!(
10661+
build_mem_used, without_null,
10662+
"fallback without the scope map"
10663+
);
1066810664
Ok(())
1066910665
}
1067010666

‎datafusion/proto/tests/cases/roundtrip_logical_plan.rs‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3916,19 +3916,6 @@ async fn roundtrip_join_null_aware() -> Result<()> {
39163916
",
39173917
(3, 2),
39183918
),
3919-
// `b` and `y` are NOT NULL, so `b = y` is a scope key, not a value key.
3920-
(
3921-
"
3922-
SELECT k
3923-
FROM (VALUES (1, 1, 2), (2, 3, 4)) AS t1(k, a, b)
3924-
WHERE (a, b) NOT IN (
3925-
SELECT x, y
3926-
FROM (VALUES (1, CAST(NULL AS INT), 2)) AS t2(k, x, y)
3927-
WHERE t2.k = t1.k
3928-
)
3929-
",
3930-
(3, 1),
3931-
),
39323919
];
39333920

39343921
for (sql, expected_keys) in cases {

‎datafusion/sqllogictest/test_files/null_aware_anti_join.slt‎

Lines changed: 86 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -410,9 +410,10 @@ NULL 0
410410
#############
411411

412412
# Results under a batch size sweep are in null_aware_multi_column.slt. A tuple
413-
# NOT IN is a null-aware join whose leading keys are the tuple elements
414-
# (`value_keys`), followed by any correlation keys. Like the scalar NOT IN,
415-
# JoinSelection swaps it into a null-aware RightAnti that builds on the
413+
# NOT IN is a null-aware join whose leading keys are the tuple elements that
414+
# can be NULL (`value_keys`), followed by the elements that cannot and by any
415+
# correlation keys, which only narrow the scope of a row. Like the scalar NOT
416+
# IN, JoinSelection swaps it into a null-aware RightAnti that builds on the
416417
# subquery side when that side is not larger, keeping the key order.
417418

418419
statement ok
@@ -604,6 +605,88 @@ physical_plan
604605
04)------DataSourceExec: partitions=1, partition_sizes=[1]
605606
05)------DataSourceExec: partitions=1, partition_sizes=[1]
606607

608+
# A correlated filter that rejects NULL makes `b = y` a scope key although
609+
# both columns are nullable (the `mc_scope` results in
610+
# null_aware_multi_column.slt).
611+
query TT
612+
EXPLAIN SELECT * FROM multi_col_outer o
613+
WHERE (a, b) NOT IN (SELECT x, y FROM multi_col_inner i WHERE i.y >= o.b);
614+
----
615+
logical_plan
616+
01)LeftAnti Join: o.a = __correlated_sq_1.x, o.b = __correlated_sq_1.y Filter: __correlated_sq_1.y >= o.b null_aware
617+
02)--SubqueryAlias: o
618+
03)----TableScan: multi_col_outer projection=[k, a, b]
619+
04)--SubqueryAlias: __correlated_sq_1
620+
05)----SubqueryAlias: i
621+
06)------TableScan: multi_col_inner projection=[x, y]
622+
physical_plan
623+
01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(x@0, a@1), (y@1, b@2)], filter=y@1 >= b@0, null_aware
624+
02)--DataSourceExec: partitions=1, partition_sizes=[1]
625+
03)--DataSourceExec: partitions=1, partition_sizes=[1]
626+
627+
# A dictionary can hold a NULL without a null bit, so its NOT NULL field does
628+
# not rule out a NULL element. A cast to a common type hides the dictionary
629+
# from the key's type, so the type of each column the key reads counts too:
630+
# `b` stays a value key whichever side is cast.
631+
statement ok
632+
CREATE TABLE multi_col_outer_dict AS
633+
SELECT k, a, arrow_cast(b, 'Dictionary(Int32, Int32)') AS b FROM multi_col_outer_nn;
634+
635+
statement ok
636+
CREATE TABLE multi_col_inner_i64 AS
637+
SELECT k, x, arrow_cast(y, 'Int64') AS y FROM multi_col_inner_nn;
638+
639+
query TT
640+
EXPLAIN SELECT * FROM multi_col_outer_dict
641+
WHERE (a, b) NOT IN (SELECT x, y FROM multi_col_inner_i64);
642+
----
643+
logical_plan
644+
01)LeftAnti Join: multi_col_outer_dict.a = __correlated_sq_1.x, CAST(multi_col_outer_dict.b AS Int64) = __correlated_sq_1.y null_aware(value_keys=2)
645+
02)--TableScan: multi_col_outer_dict projection=[k, a, b]
646+
03)--SubqueryAlias: __correlated_sq_1
647+
04)----TableScan: multi_col_inner_i64 projection=[x, y]
648+
physical_plan
649+
01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(x@0, a@1), (y@1, CAST(multi_col_outer_dict.b AS Int64)@3)], projection=[k@0, a@1, b@2], null_aware(value_keys=2)
650+
02)--DataSourceExec: partitions=1, partition_sizes=[1]
651+
03)--ProjectionExec: expr=[k@0 as k, a@1 as a, b@2 as b, CAST(b@2 AS Int64) as CAST(multi_col_outer_dict.b AS Int64)]
652+
04)----DataSourceExec: partitions=1, partition_sizes=[1]
653+
654+
statement ok
655+
CREATE TABLE multi_col_outer_i64 AS
656+
SELECT k, a, arrow_cast(b, 'Int64') AS b FROM multi_col_outer_nn;
657+
658+
statement ok
659+
CREATE TABLE multi_col_inner_dict AS
660+
SELECT k, x, arrow_cast(y, 'Dictionary(Int32, Int32)') AS y FROM multi_col_inner_nn;
661+
662+
query TT
663+
EXPLAIN SELECT * FROM multi_col_outer_i64
664+
WHERE (a, b) NOT IN (SELECT x, y FROM multi_col_inner_dict);
665+
----
666+
logical_plan
667+
01)LeftAnti Join: multi_col_outer_i64.a = __correlated_sq_1.x, multi_col_outer_i64.b = __correlated_sq_1.multi_col_inner_dict.y null_aware(value_keys=2)
668+
02)--TableScan: multi_col_outer_i64 projection=[k, a, b]
669+
03)--SubqueryAlias: __correlated_sq_1
670+
04)----Projection: multi_col_inner_dict.x, CAST(multi_col_inner_dict.y AS Int64)
671+
05)------TableScan: multi_col_inner_dict projection=[x, y]
672+
physical_plan
673+
01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(x@0, a@1), (multi_col_inner_dict.y@1, b@2)], null_aware(value_keys=2)
674+
02)--ProjectionExec: expr=[x@0 as x, CAST(y@1 AS Int64) as multi_col_inner_dict.y]
675+
03)----DataSourceExec: partitions=1, partition_sizes=[1]
676+
04)--DataSourceExec: partitions=1, partition_sizes=[1]
677+
678+
statement ok
679+
DROP TABLE multi_col_outer_dict;
680+
681+
statement ok
682+
DROP TABLE multi_col_inner_i64;
683+
684+
statement ok
685+
DROP TABLE multi_col_outer_i64;
686+
687+
statement ok
688+
DROP TABLE multi_col_inner_dict;
689+
607690
statement ok
608691
DROP TABLE multi_col_outer_nn;
609692

‎datafusion/sqllogictest/test_files/null_aware_multi_column.slt‎

Lines changed: 0 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -579,25 +579,6 @@ r
579579
s
580580
u
581581

582-
# The NOT NULL element first gives the same rows.
583-
query T rowsort
584-
SELECT value FROM mc_nn_outer
585-
WHERE (b, a) NOT IN (SELECT y, x FROM mc_nn_inner);
586-
----
587-
r
588-
s
589-
u
590-
591-
# With one nullable element the tuple is a scalar NOT IN on it, correlated by
592-
# the other elements.
593-
query T rowsort
594-
SELECT value FROM mc_nn_outer
595-
WHERE a NOT IN (SELECT x FROM mc_nn_inner WHERE y = b);
596-
----
597-
r
598-
s
599-
u
600-
601582
# With a correlation, t's scope is k = 2, {(8, 9)}, which it mismatches.
602583
query T rowsort
603584
SELECT value FROM mc_nn_outer o
@@ -608,16 +589,6 @@ s
608589
t
609590
u
610591

611-
# The correlation as a NOT NULL tuple element gives the same rows.
612-
query T rowsort
613-
SELECT value FROM mc_nn_outer
614-
WHERE (a, k, b) NOT IN (SELECT x, k, y FROM mc_nn_inner);
615-
----
616-
r
617-
s
618-
t
619-
u
620-
621592
# Two nullable elements and a scope key. The subquery gives (1, 2, 1),
622593
# (NULL, 4, 1) and (8, 9, NULL): t's k = 2 and u's a = 7 are definite
623594
# mismatches.
@@ -639,19 +610,6 @@ s
639610
u
640611
v
641612

642-
# An empty subquery keeps every row, NULLs included.
643-
query T rowsort
644-
SELECT value FROM mc_nn_outer
645-
WHERE (a, b) NOT IN (SELECT x, y FROM mc_nn_inner WHERE x > 100);
646-
----
647-
p
648-
q
649-
r
650-
s
651-
t
652-
u
653-
v
654-
655613
# A RunEndEncoded element is not hashable. A run of NULLs has no null bit, so
656614
# a field declared NOT NULL does not rule out a NULL element, and the element
657615
# is treated like one that can be NULL: it is rejected (see above) rather than
@@ -695,15 +653,6 @@ t true
695653
u true
696654
v NULL
697655

698-
query T rowsort
699-
SELECT value FROM mc_nn_outer
700-
WHERE (a, b) NOT IN (SELECT x, y FROM mc_nn_inner) OR value = 'q';
701-
----
702-
q
703-
r
704-
s
705-
u
706-
707656
# An element that can be NULL in the schema but not in scope: the correlated
708657
# filter `i.y >= o.b` rejects a NULL `y` or `b`, so `b = y` is a scope key, and
709658
# the rows with a NULL `b` reach the join with a NULL scope key. Such a row has

0 commit comments

Comments
 (0)