@@ -563,3 +563,177 @@ ORDER BY start_timestamp, trace_id
563563LIMIT 1;
564564----
5655652024-10-01T00:00:00
566+
567+ ###
568+ # Array function predicate pushdown tests
569+ # These tests verify that array_has, array_has_all, and array_has_any predicates
570+ # are correctly pushed down to the DataSourceExec node
571+ ###
572+
573+ # Create test data with array columns
574+ statement ok
575+ COPY (
576+ SELECT 1 as id, ['rust', 'performance'] as tags
577+ UNION ALL
578+ SELECT 2 as id, ['python', 'javascript'] as tags
579+ UNION ALL
580+ SELECT 3 as id, ['rust', 'webassembly'] as tags
581+ )
582+ TO 'test_files/scratch/parquet_filter_pushdown/array_data/data.parquet';
583+
584+ statement ok
585+ CREATE EXTERNAL TABLE array_test STORED AS PARQUET LOCATION 'test_files/scratch/parquet_filter_pushdown/array_data/';
586+
587+ statement ok
588+ SET datafusion.execution.parquet.pushdown_filters = true;
589+
590+ # Test array_has predicate pushdown
591+ query I?
592+ SELECT id, tags FROM array_test WHERE array_has(tags, 'rust') ORDER BY id;
593+ ----
594+ 1 [rust, performance]
595+ 3 [rust, webassembly]
596+
597+ query TT
598+ EXPLAIN SELECT id, tags FROM array_test WHERE array_has(tags, 'rust') ORDER BY id;
599+ ----
600+ logical_plan
601+ 01)Sort: array_test.id ASC NULLS LAST
602+ 02)--Filter: array_has(array_test.tags, Utf8("rust"))
603+ 03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has(array_test.tags, Utf8("rust"))]
604+ physical_plan
605+ 01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
606+ 02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has(tags@1, rust)
607+
608+ # Test array_has_all predicate pushdown
609+ query I?
610+ SELECT id, tags FROM array_test WHERE array_has_all(tags, ['rust', 'performance']) ORDER BY id;
611+ ----
612+ 1 [rust, performance]
613+
614+ query TT
615+ EXPLAIN SELECT id, tags FROM array_test WHERE array_has_all(tags, ['rust', 'performance']) ORDER BY id;
616+ ----
617+ logical_plan
618+ 01)Sort: array_test.id ASC NULLS LAST
619+ 02)--Filter: array_has_all(array_test.tags, List([rust, performance]))
620+ 03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has_all(array_test.tags, List([rust, performance]))]
621+ physical_plan
622+ 01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
623+ 02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_all(tags@1, [rust, performance])
624+
625+ # Test array_has_any predicate pushdown
626+ query I?
627+ SELECT id, tags FROM array_test WHERE array_has_any(tags, ['python', 'go']) ORDER BY id;
628+ ----
629+ 2 [python, javascript]
630+
631+ query TT
632+ EXPLAIN SELECT id, tags FROM array_test WHERE array_has_any(tags, ['python', 'go']) ORDER BY id;
633+ ----
634+ logical_plan
635+ 01)Sort: array_test.id ASC NULLS LAST
636+ 02)--Filter: array_has_any(array_test.tags, List([python, go]))
637+ 03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has_any(array_test.tags, List([python, go]))]
638+ physical_plan
639+ 01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
640+ 02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_any(tags@1, [python, go])
641+
642+ # Test complex predicate with OR
643+ query I?
644+ SELECT id, tags FROM array_test WHERE array_has_all(tags, ['rust']) OR array_has_any(tags, ['python', 'go']) ORDER BY id;
645+ ----
646+ 1 [rust, performance]
647+ 2 [python, javascript]
648+ 3 [rust, webassembly]
649+
650+ query TT
651+ EXPLAIN SELECT id, tags FROM array_test WHERE array_has_all(tags, ['rust']) OR array_has_any(tags, ['python', 'go']) ORDER BY id;
652+ ----
653+ logical_plan
654+ 01)Sort: array_test.id ASC NULLS LAST
655+ 02)--Filter: array_has_all(array_test.tags, List([rust])) OR array_has_any(array_test.tags, List([python, go]))
656+ 03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has_all(array_test.tags, List([rust])) OR array_has_any(array_test.tags, List([python, go]))]
657+ physical_plan
658+ 01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
659+ 02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_all(tags@1, [rust]) OR array_has_any(tags@1, [python, go])
660+
661+ # Test array function with other predicates
662+ query I?
663+ SELECT id, tags FROM array_test WHERE id > 1 AND array_has(tags, 'rust') ORDER BY id;
664+ ----
665+ 3 [rust, webassembly]
666+
667+ query TT
668+ EXPLAIN SELECT id, tags FROM array_test WHERE id > 1 AND array_has(tags, 'rust') ORDER BY id;
669+ ----
670+ logical_plan
671+ 01)Sort: array_test.id ASC NULLS LAST
672+ 02)--Filter: array_test.id > Int64(1) AND array_has(array_test.tags, Utf8("rust"))
673+ 03)----TableScan: array_test projection=[id, tags], partial_filters=[array_test.id > Int64(1), array_has(array_test.tags, Utf8("rust"))]
674+ physical_plan
675+ 01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
676+ 02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=id@0 > 1 AND array_has(tags@1, rust), pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 1, required_guarantees=[]
677+
678+ ###
679+ # Test filter pushdown through UNION with mixed support
680+ # This tests the case where one child supports filter pushdown (parquet) and one doesn't (memory table)
681+ ###
682+
683+ # enable filter pushdown
684+ statement ok
685+ set datafusion.execution.parquet.pushdown_filters = true;
686+
687+ statement ok
688+ set datafusion.optimizer.max_passes = 0;
689+
690+ # Create memory table with matching schema (a: VARCHAR, b: BIGINT)
691+ statement ok
692+ CREATE TABLE t_union_mem(a VARCHAR, b BIGINT) AS VALUES ('qux', 4), ('quux', 5);
693+
694+ # Create parquet table with matching schema
695+ statement ok
696+ CREATE EXTERNAL TABLE t_union_parquet(a VARCHAR, b BIGINT) STORED AS PARQUET
697+ LOCATION 'test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet';
698+
699+ # Query results combining memory table and Parquet with filter
700+ query I rowsort
701+ SELECT b FROM (
702+ SELECT a, b FROM t_union_mem
703+ UNION ALL
704+ SELECT a, b FROM t_union_parquet
705+ ) WHERE b > 2;
706+ ----
707+ 3
708+ 4
709+ 5
710+ 50
711+
712+ # Explain the union query - filter should be pushed to parquet but not memory table
713+ query TT
714+ EXPLAIN SELECT b FROM (
715+ SELECT a, b FROM t_union_mem
716+ UNION ALL
717+ SELECT a, b FROM t_union_parquet
718+ ) WHERE b > 2;
719+ ----
720+ logical_plan
721+ 01)Projection: b
722+ 02)--Filter: b > Int64(2)
723+ 03)----Union
724+ 04)------Projection: t_union_mem.a, t_union_mem.b
725+ 05)--------TableScan: t_union_mem
726+ 06)------Projection: t_union_parquet.a, t_union_parquet.b
727+ 07)--------TableScan: t_union_parquet
728+ physical_plan
729+ 01)UnionExec
730+ 02)--FilterExec: b@0 > 2
731+ 03)----DataSourceExec: partitions=1, partition_sizes=[1]
732+ 04)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet]]}, projection=[b], file_type=parquet, predicate=b@1 > 2, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[]
733+
734+ # Clean up union test tables
735+ statement ok
736+ DROP TABLE t_union_mem;
737+
738+ statement ok
739+ DROP TABLE t_union_parquet;
0 commit comments