Skip to content

Commit a6d5391

Browse files
gruuyaclaude
andcommitted
Fix CASE full-match inference and skip unused pruning state
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
1 parent 28b42fe commit a6d5391

3 files changed

Lines changed: 102 additions & 2 deletions

File tree

‎datafusion/physical-plan/src/joins/hash_join/exec.rs‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3085,7 +3085,9 @@ async fn collect_left_input(
30853085

30863086
let map = Arc::new(join_hash_map);
30873087

3088-
let membership = if num_rows == 0 {
3088+
// Nothing reads the strategy unless the dynamic filter accumulator exists,
3089+
// and that exists only when the pushdown is enabled.
3090+
let membership = if num_rows == 0 || !should_compute_dynamic_filters {
30893091
PushdownStrategy::Empty
30903092
} else {
30913093
// If the build side is small enough we can use IN list pushdown.
@@ -3287,6 +3289,53 @@ mod tests {
32873289
Ok(())
32883290
}
32893291

3292+
#[tokio::test]
3293+
async fn no_pruning_state_without_dynamic_filters() -> Result<()> {
3294+
let schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Int64, false)]));
3295+
let build = RecordBatch::try_new(
3296+
Arc::clone(&schema),
3297+
vec![Arc::new(Int64Array::from_iter_values(
3298+
(0..151).map(|i| i * 10_000),
3299+
))],
3300+
)?;
3301+
let probe = RecordBatch::try_new(
3302+
Arc::clone(&schema),
3303+
vec![Arc::new(Int64Array::from_iter_values([
3304+
0, 500_000, 1_500_000,
3305+
]))],
3306+
)?;
3307+
let on = vec![(
3308+
Arc::new(Column::new_with_schema("k", &schema)?) as _,
3309+
Arc::new(Column::new_with_schema("k", &schema)?) as _,
3310+
)];
3311+
let join = join(
3312+
TestMemoryExec::try_new_exec(&[vec![build]], Arc::clone(&schema), None)?,
3313+
TestMemoryExec::try_new_exec(&[vec![probe]], Arc::clone(&schema), None)?,
3314+
on,
3315+
&JoinType::Inner,
3316+
NullEquality::NullEqualsNothing,
3317+
)?;
3318+
3319+
// Bounds are still collected to test perfect-hash-join candidacy, so 151
3320+
// keys over 1.5M would size a bitmap at the 128 KiB cap - far past this.
3321+
let runtime = RuntimeEnvBuilder::new()
3322+
.with_memory_limit(100_000, 1.0)
3323+
.build_arc()?;
3324+
let mut config = SessionConfig::new();
3325+
config
3326+
.options_mut()
3327+
.optimizer
3328+
.enable_join_dynamic_filter_pushdown = false;
3329+
let task_ctx = Arc::new(
3330+
TaskContext::default()
3331+
.with_runtime(runtime)
3332+
.with_session_config(config),
3333+
);
3334+
let batches = common::collect(join.execute(0, task_ctx)?).await?;
3335+
assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 3);
3336+
Ok(())
3337+
}
3338+
32903339
#[derive(Debug)]
32913340
struct PartitionedTestExec {
32923341
cache: Arc<PlanProperties>,

‎datafusion/pruning/src/pruning_predicate.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1872,7 +1872,11 @@ fn build_predicate_expression(
18721872
if !matches!(case.data_type(schema), Ok(DataType::Boolean)) {
18731873
return unhandled_hook.handle(expr);
18741874
}
1875-
// A missing `ELSE` yields NULL, which never matches, so it adds nothing.
1875+
// Without an `ELSE`, unmatched rows are UNKNOWN: not a match, but not
1876+
// FALSE either, which full-match inference must be able to tell apart.
1877+
if case.else_expr().is_none() {
1878+
return unhandled_hook.handle(expr);
1879+
}
18761880
return case
18771881
.when_then_expr()
18781882
.iter()

‎datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1052,3 +1052,50 @@ set datafusion.execution.parquet.reorder_filters = false;
10521052

10531053
statement ok
10541054
DROP TABLE dict_filter_bug;
1055+
1056+
# Regression: a `CASE` with no `ELSE` leaves unmatched rows UNKNOWN, not FALSE.
1057+
# Treating them as FALSE lets the inverse predicate call a row group fully
1058+
# matched and skip the row filter. Required columns admit no `IS NULL` rescue.
1059+
statement ok
1060+
set datafusion.execution.parquet.pushdown_filters = true;
1061+
1062+
statement ok
1063+
COPY (SELECT column1 AS a, column2 AS b FROM (VALUES (2, 1), (1, 2)))
1064+
TO 'test_files/scratch/parquet_filter_pushdown/case_without_else.parquet'
1065+
STORED AS PARQUET
1066+
OPTIONS ('format.max_row_group_size' '1');
1067+
1068+
statement ok
1069+
CREATE EXTERNAL TABLE case_without_else (a BIGINT NOT NULL, b BIGINT NOT NULL)
1070+
STORED AS PARQUET
1071+
LOCATION 'test_files/scratch/parquet_filter_pushdown/case_without_else.parquet';
1072+
1073+
query II
1074+
SELECT * FROM case_without_else
1075+
WHERE NOT (CASE WHEN a = 1 THEN false END) AND b > 0 ORDER BY a;
1076+
----
1077+
1 2
1078+
1079+
query II
1080+
SELECT * FROM case_without_else
1081+
WHERE NOT (CASE WHEN a = 1 THEN false END) AND b > 0 LIMIT 1;
1082+
----
1083+
1 2
1084+
1085+
query II
1086+
SELECT * FROM case_without_else
1087+
WHERE NOT (CASE WHEN a = 1 THEN false ELSE NULL END) AND b > 0 ORDER BY a;
1088+
----
1089+
1 2
1090+
1091+
query II
1092+
SELECT * FROM case_without_else
1093+
WHERE NOT (CASE WHEN a = 1 THEN NULL ELSE false END) AND b > 0 ORDER BY a;
1094+
----
1095+
2 1
1096+
1097+
statement ok
1098+
set datafusion.execution.parquet.pushdown_filters = false;
1099+
1100+
statement ok
1101+
DROP TABLE case_without_else;

0 commit comments

Comments
 (0)