Skip to content

Commit a9b5682

Browse files
committed
feat: Pushdown filters through UnionExec nodes (apache#20145)
- Closes apache#20144 see apache#20144 This PR impl `handle_child_pushdown_result` for `UnionExec`, for any case, the filter will always pushdown to UnionExec UnionExec needs specialized filter pushdown handling when children have heterogeneous pushdown support. Without this, when some children support pushdown and others don't, the default behavior would leave FilterExec above UnionExec, re-applying filters to outputs of all children—including those that already applied the filters via pushdown. This specialized implementation adds FilterExec only to children that don't support pushdown, avoiding redundant filtering and improving performance. ``` Example: Given Child1 (no pushdown support) and Child2 (has pushdown support) Default behavior: This implementation: FilterExec UnionExec UnionExec FilterExec Child1 Child1 Child2(filter) Child2(filter) ``` yes, add two test cases
1 parent 7ce4be0 commit a9b5682

3 files changed

Lines changed: 320 additions & 2 deletions

File tree

‎datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1894,6 +1894,67 @@ fn test_filter_pushdown_through_union() {
18941894
);
18951895
}
18961896

1897+
#[test]
1898+
fn test_filter_pushdown_through_union_mixed_support() {
1899+
// Test case where one child supports filter pushdown and one doesn't
1900+
let scan1 = TestScanBuilder::new(schema()).with_support(true).build();
1901+
let scan2 = TestScanBuilder::new(schema()).with_support(false).build();
1902+
1903+
let union = UnionExec::try_new(vec![scan1, scan2]).unwrap();
1904+
1905+
let predicate = col_lit_predicate("a", "foo", &schema());
1906+
let plan = Arc::new(FilterExec::try_new(predicate, union).unwrap());
1907+
1908+
insta::assert_snapshot!(
1909+
OptimizationTest::new(plan, FilterPushdown::new(), true),
1910+
@r"
1911+
OptimizationTest:
1912+
input:
1913+
- FilterExec: a@0 = foo
1914+
- UnionExec
1915+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
1916+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
1917+
output:
1918+
Ok:
1919+
- UnionExec
1920+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=a@0 = foo
1921+
- FilterExec: a@0 = foo
1922+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
1923+
"
1924+
);
1925+
}
1926+
1927+
#[test]
1928+
fn test_filter_pushdown_through_union_does_not_support() {
1929+
// Test case where one child supports filter pushdown and one doesn't
1930+
let scan1 = TestScanBuilder::new(schema()).with_support(false).build();
1931+
let scan2 = TestScanBuilder::new(schema()).with_support(false).build();
1932+
1933+
let union = UnionExec::try_new(vec![scan1, scan2]).unwrap();
1934+
1935+
let predicate = col_lit_predicate("a", "foo", &schema());
1936+
let plan = Arc::new(FilterExec::try_new(predicate, union).unwrap());
1937+
1938+
insta::assert_snapshot!(
1939+
OptimizationTest::new(plan, FilterPushdown::new(), true),
1940+
@"
1941+
OptimizationTest:
1942+
input:
1943+
- FilterExec: a@0 = foo
1944+
- UnionExec
1945+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
1946+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
1947+
output:
1948+
Ok:
1949+
- UnionExec
1950+
- FilterExec: a@0 = foo
1951+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
1952+
- FilterExec: a@0 = foo
1953+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
1954+
"
1955+
);
1956+
}
1957+
18971958
/// Schema:
18981959
/// a: String
18991960
/// b: String

‎datafusion/physical-plan/src/union.rs‎

Lines changed: 85 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,11 @@ use crate::execution_plan::{
3636
InvariantLevel, boundedness_from_children, check_default_invariants,
3737
emission_type_from_children,
3838
};
39-
use crate::filter_pushdown::{FilterDescription, FilterPushdownPhase};
39+
use crate::filter::FilterExec;
40+
use crate::filter_pushdown::{
41+
ChildPushdownResult, FilterDescription, FilterPushdownPhase,
42+
FilterPushdownPropagation, PushedDown,
43+
};
4044
use crate::metrics::BaselineMetrics;
4145
use crate::projection::{ProjectionExec, make_with_child};
4246
use crate::stream::ObservedStream;
@@ -49,7 +53,9 @@ use datafusion_common::{
4953
Result, assert_or_internal_err, exec_err, internal_datafusion_err,
5054
};
5155
use datafusion_execution::TaskContext;
52-
use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr, calculate_union};
56+
use datafusion_physical_expr::{
57+
EquivalenceProperties, PhysicalExpr, calculate_union, conjunction,
58+
};
5359

5460
use futures::Stream;
5561
use itertools::Itertools;
@@ -370,6 +376,83 @@ impl ExecutionPlan for UnionExec {
370376
) -> Result<FilterDescription> {
371377
FilterDescription::from_children(parent_filters, &self.children())
372378
}
379+
380+
fn handle_child_pushdown_result(
381+
&self,
382+
phase: FilterPushdownPhase,
383+
child_pushdown_result: ChildPushdownResult,
384+
_config: &ConfigOptions,
385+
) -> Result<FilterPushdownPropagation<Arc<dyn ExecutionPlan>>> {
386+
// Pre phase: handle heterogeneous pushdown by wrapping individual
387+
// children with FilterExec and reporting all filters as handled.
388+
// Post phase: use default behavior to let the filter creator decide how to handle
389+
// filters that weren't fully pushed down.
390+
if !matches!(phase, FilterPushdownPhase::Pre) {
391+
return Ok(FilterPushdownPropagation::if_all(child_pushdown_result));
392+
}
393+
394+
// UnionExec needs specialized filter pushdown handling when children have
395+
// heterogeneous pushdown support. Without this, when some children support
396+
// pushdown and others don't, the default behavior would leave FilterExec
397+
// above UnionExec, re-applying filters to outputs of all children—including
398+
// those that already applied the filters via pushdown. This specialized
399+
// implementation adds FilterExec only to children that don't support
400+
// pushdown, avoiding redundant filtering and improving performance.
401+
//
402+
// Example: Given Child1 (no pushdown support) and Child2 (has pushdown support)
403+
// Default behavior: This implementation:
404+
// FilterExec UnionExec
405+
// UnionExec FilterExec
406+
// Child1 Child1
407+
// Child2(filter) Child2(filter)
408+
409+
// Collect unsupported filters for each child
410+
let mut unsupported_filters_per_child = vec![Vec::new(); self.inputs.len()];
411+
for parent_filter_result in child_pushdown_result.parent_filters.iter() {
412+
for (child_idx, &child_result) in
413+
parent_filter_result.child_results.iter().enumerate()
414+
{
415+
if matches!(child_result, PushedDown::No) {
416+
unsupported_filters_per_child[child_idx]
417+
.push(Arc::clone(&parent_filter_result.filter));
418+
}
419+
}
420+
}
421+
422+
// Wrap children that have unsupported filters with FilterExec
423+
let mut new_children = self.inputs.clone();
424+
for (child_idx, unsupported_filters) in
425+
unsupported_filters_per_child.iter().enumerate()
426+
{
427+
if !unsupported_filters.is_empty() {
428+
let combined_filter = conjunction(unsupported_filters.clone());
429+
new_children[child_idx] = Arc::new(FilterExec::try_new(
430+
combined_filter,
431+
Arc::clone(&self.inputs[child_idx]),
432+
)?);
433+
}
434+
}
435+
436+
// Check if any children were modified
437+
let children_modified = new_children
438+
.iter()
439+
.zip(self.inputs.iter())
440+
.any(|(new, old)| !Arc::ptr_eq(new, old));
441+
442+
let all_filters_pushed =
443+
vec![PushedDown::Yes; child_pushdown_result.parent_filters.len()];
444+
let propagation = if children_modified {
445+
let updated_node = UnionExec::try_new(new_children)?;
446+
FilterPushdownPropagation::with_parent_pushdown_result(all_filters_pushed)
447+
.with_updated_node(updated_node)
448+
} else {
449+
FilterPushdownPropagation::with_parent_pushdown_result(all_filters_pushed)
450+
};
451+
452+
// Report all parent filters as supported since we've ensured they're applied
453+
// on all children (either pushed down or via FilterExec)
454+
Ok(propagation)
455+
}
373456
}
374457

375458
/// Combines multiple input streams by interleaving them.

‎datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -563,3 +563,177 @@ ORDER BY start_timestamp, trace_id
563563
LIMIT 1;
564564
----
565565
2024-10-01T00:00:00
566+
567+
###
568+
# Array function predicate pushdown tests
569+
# These tests verify that array_has, array_has_all, and array_has_any predicates
570+
# are correctly pushed down to the DataSourceExec node
571+
###
572+
573+
# Create test data with array columns
574+
statement ok
575+
COPY (
576+
SELECT 1 as id, ['rust', 'performance'] as tags
577+
UNION ALL
578+
SELECT 2 as id, ['python', 'javascript'] as tags
579+
UNION ALL
580+
SELECT 3 as id, ['rust', 'webassembly'] as tags
581+
)
582+
TO 'test_files/scratch/parquet_filter_pushdown/array_data/data.parquet';
583+
584+
statement ok
585+
CREATE EXTERNAL TABLE array_test STORED AS PARQUET LOCATION 'test_files/scratch/parquet_filter_pushdown/array_data/';
586+
587+
statement ok
588+
SET datafusion.execution.parquet.pushdown_filters = true;
589+
590+
# Test array_has predicate pushdown
591+
query I?
592+
SELECT id, tags FROM array_test WHERE array_has(tags, 'rust') ORDER BY id;
593+
----
594+
1 [rust, performance]
595+
3 [rust, webassembly]
596+
597+
query TT
598+
EXPLAIN SELECT id, tags FROM array_test WHERE array_has(tags, 'rust') ORDER BY id;
599+
----
600+
logical_plan
601+
01)Sort: array_test.id ASC NULLS LAST
602+
02)--Filter: array_has(array_test.tags, Utf8("rust"))
603+
03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has(array_test.tags, Utf8("rust"))]
604+
physical_plan
605+
01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
606+
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has(tags@1, rust)
607+
608+
# Test array_has_all predicate pushdown
609+
query I?
610+
SELECT id, tags FROM array_test WHERE array_has_all(tags, ['rust', 'performance']) ORDER BY id;
611+
----
612+
1 [rust, performance]
613+
614+
query TT
615+
EXPLAIN SELECT id, tags FROM array_test WHERE array_has_all(tags, ['rust', 'performance']) ORDER BY id;
616+
----
617+
logical_plan
618+
01)Sort: array_test.id ASC NULLS LAST
619+
02)--Filter: array_has_all(array_test.tags, List([rust, performance]))
620+
03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has_all(array_test.tags, List([rust, performance]))]
621+
physical_plan
622+
01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
623+
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_all(tags@1, [rust, performance])
624+
625+
# Test array_has_any predicate pushdown
626+
query I?
627+
SELECT id, tags FROM array_test WHERE array_has_any(tags, ['python', 'go']) ORDER BY id;
628+
----
629+
2 [python, javascript]
630+
631+
query TT
632+
EXPLAIN SELECT id, tags FROM array_test WHERE array_has_any(tags, ['python', 'go']) ORDER BY id;
633+
----
634+
logical_plan
635+
01)Sort: array_test.id ASC NULLS LAST
636+
02)--Filter: array_has_any(array_test.tags, List([python, go]))
637+
03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has_any(array_test.tags, List([python, go]))]
638+
physical_plan
639+
01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
640+
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_any(tags@1, [python, go])
641+
642+
# Test complex predicate with OR
643+
query I?
644+
SELECT id, tags FROM array_test WHERE array_has_all(tags, ['rust']) OR array_has_any(tags, ['python', 'go']) ORDER BY id;
645+
----
646+
1 [rust, performance]
647+
2 [python, javascript]
648+
3 [rust, webassembly]
649+
650+
query TT
651+
EXPLAIN SELECT id, tags FROM array_test WHERE array_has_all(tags, ['rust']) OR array_has_any(tags, ['python', 'go']) ORDER BY id;
652+
----
653+
logical_plan
654+
01)Sort: array_test.id ASC NULLS LAST
655+
02)--Filter: array_has_all(array_test.tags, List([rust])) OR array_has_any(array_test.tags, List([python, go]))
656+
03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has_all(array_test.tags, List([rust])) OR array_has_any(array_test.tags, List([python, go]))]
657+
physical_plan
658+
01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
659+
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_all(tags@1, [rust]) OR array_has_any(tags@1, [python, go])
660+
661+
# Test array function with other predicates
662+
query I?
663+
SELECT id, tags FROM array_test WHERE id > 1 AND array_has(tags, 'rust') ORDER BY id;
664+
----
665+
3 [rust, webassembly]
666+
667+
query TT
668+
EXPLAIN SELECT id, tags FROM array_test WHERE id > 1 AND array_has(tags, 'rust') ORDER BY id;
669+
----
670+
logical_plan
671+
01)Sort: array_test.id ASC NULLS LAST
672+
02)--Filter: array_test.id > Int64(1) AND array_has(array_test.tags, Utf8("rust"))
673+
03)----TableScan: array_test projection=[id, tags], partial_filters=[array_test.id > Int64(1), array_has(array_test.tags, Utf8("rust"))]
674+
physical_plan
675+
01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
676+
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=id@0 > 1 AND array_has(tags@1, rust), pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 1, required_guarantees=[]
677+
678+
###
679+
# Test filter pushdown through UNION with mixed support
680+
# This tests the case where one child supports filter pushdown (parquet) and one doesn't (memory table)
681+
###
682+
683+
# enable filter pushdown
684+
statement ok
685+
set datafusion.execution.parquet.pushdown_filters = true;
686+
687+
statement ok
688+
set datafusion.optimizer.max_passes = 0;
689+
690+
# Create memory table with matching schema (a: VARCHAR, b: BIGINT)
691+
statement ok
692+
CREATE TABLE t_union_mem(a VARCHAR, b BIGINT) AS VALUES ('qux', 4), ('quux', 5);
693+
694+
# Create parquet table with matching schema
695+
statement ok
696+
CREATE EXTERNAL TABLE t_union_parquet(a VARCHAR, b BIGINT) STORED AS PARQUET
697+
LOCATION 'test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet';
698+
699+
# Query results combining memory table and Parquet with filter
700+
query I rowsort
701+
SELECT b FROM (
702+
SELECT a, b FROM t_union_mem
703+
UNION ALL
704+
SELECT a, b FROM t_union_parquet
705+
) WHERE b > 2;
706+
----
707+
3
708+
4
709+
5
710+
50
711+
712+
# Explain the union query - filter should be pushed to parquet but not memory table
713+
query TT
714+
EXPLAIN SELECT b FROM (
715+
SELECT a, b FROM t_union_mem
716+
UNION ALL
717+
SELECT a, b FROM t_union_parquet
718+
) WHERE b > 2;
719+
----
720+
logical_plan
721+
01)Projection: b
722+
02)--Filter: b > Int64(2)
723+
03)----Union
724+
04)------Projection: t_union_mem.a, t_union_mem.b
725+
05)--------TableScan: t_union_mem
726+
06)------Projection: t_union_parquet.a, t_union_parquet.b
727+
07)--------TableScan: t_union_parquet
728+
physical_plan
729+
01)UnionExec
730+
02)--FilterExec: b@0 > 2
731+
03)----DataSourceExec: partitions=1, partition_sizes=[1]
732+
04)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet]]}, projection=[b], file_type=parquet, predicate=b@1 > 2, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[]
733+
734+
# Clean up union test tables
735+
statement ok
736+
DROP TABLE t_union_mem;
737+
738+
statement ok
739+
DROP TABLE t_union_parquet;

0 commit comments

Comments
 (0)