@@ -59,6 +59,7 @@ use datafusion_physical_plan::{
5959 coalesce_partitions:: CoalescePartitionsExec ,
6060 collect,
6161 filter:: { FilterExec , FilterExecBuilder } ,
62+ projection:: ProjectionExec ,
6263 repartition:: RepartitionExec ,
6364 sorts:: sort:: SortExec ,
6465} ;
@@ -1826,6 +1827,234 @@ fn schema() -> SchemaRef {
18261827 Arc :: clone ( & TEST_SCHEMA )
18271828}
18281829
1830+ struct ProjectionDynFilterTestCase {
1831+ schema : SchemaRef ,
1832+ batches : Vec < RecordBatch > ,
1833+ projection : Vec < ( Arc < dyn PhysicalExpr > , String ) > ,
1834+ sort_expr : PhysicalSortExpr ,
1835+ expected_plans : Vec < String > ,
1836+ }
1837+
1838+ async fn run_projection_dyn_filter_case ( case : ProjectionDynFilterTestCase ) {
1839+ let ProjectionDynFilterTestCase {
1840+ schema,
1841+ batches,
1842+ projection,
1843+ sort_expr,
1844+ expected_plans,
1845+ } = case;
1846+
1847+ let scan = TestScanBuilder :: new ( Arc :: clone ( & schema) )
1848+ . with_support ( true )
1849+ . with_batches ( batches)
1850+ . build ( ) ;
1851+
1852+ let projection_exec = Arc :: new ( ProjectionExec :: try_new ( projection, scan) . unwrap ( ) ) ;
1853+
1854+ let sort = Arc :: new (
1855+ SortExec :: new ( LexOrdering :: new ( vec ! [ sort_expr] ) . unwrap ( ) , projection_exec)
1856+ . with_fetch ( Some ( 2 ) ) ,
1857+ ) as Arc < dyn ExecutionPlan > ;
1858+
1859+ let mut config = ConfigOptions :: default ( ) ;
1860+ config. execution . parquet . pushdown_filters = true ;
1861+ config. optimizer . enable_dynamic_filter_pushdown = true ;
1862+
1863+ let optimized_plan = FilterPushdown :: new_post_optimization ( )
1864+ . optimize ( Arc :: clone ( & sort) , & config)
1865+ . unwrap ( ) ;
1866+
1867+ pretty_assertions:: assert_eq!(
1868+ format_plan_for_test( & optimized_plan) . trim( ) ,
1869+ expected_plans[ 0 ] . trim( )
1870+ ) ;
1871+
1872+ let config = SessionConfig :: new ( ) . with_batch_size ( 2 ) ;
1873+ let session_ctx = SessionContext :: new_with_config ( config) ;
1874+ session_ctx. register_object_store (
1875+ ObjectStoreUrl :: parse ( "test://" ) . unwrap ( ) . as_ref ( ) ,
1876+ Arc :: new ( InMemory :: new ( ) ) ,
1877+ ) ;
1878+ let state = session_ctx. state ( ) ;
1879+ let task_ctx = state. task_ctx ( ) ;
1880+ let mut stream = optimized_plan. execute ( 0 , Arc :: clone ( & task_ctx) ) . unwrap ( ) ;
1881+ for ( idx, expected_plan) in expected_plans. iter ( ) . enumerate ( ) . skip ( 1 ) {
1882+ stream. next ( ) . await . unwrap ( ) . unwrap ( ) ;
1883+ let formatted_plan = format_plan_for_test ( & optimized_plan) ;
1884+ pretty_assertions:: assert_eq!(
1885+ formatted_plan. trim( ) ,
1886+ expected_plan. trim( ) ,
1887+ "Mismatch at iteration {}" ,
1888+ idx
1889+ ) ;
1890+ }
1891+ }
1892+
1893+ #[ tokio:: test]
1894+ async fn test_topk_with_projection_transformation_on_dyn_filter ( ) {
1895+ let schema = Arc :: new ( Schema :: new ( vec ! [
1896+ Field :: new( "a" , DataType :: Int32 , false ) ,
1897+ Field :: new( "b" , DataType :: Utf8 , false ) ,
1898+ Field :: new( "c" , DataType :: Float64 , false ) ,
1899+ ] ) ) ;
1900+ let simple_abc = vec ! [
1901+ record_batch!(
1902+ ( "a" , Int32 , [ 1 , 2 , 3 ] ) ,
1903+ ( "b" , Utf8 , [ "x" , "y" , "z" ] ) ,
1904+ ( "c" , Float64 , [ 1.0 , 2.0 , 3.0 ] )
1905+ )
1906+ . unwrap( ) ,
1907+ ] ;
1908+
1909+ // Case 1: Reordering [b, a]
1910+ run_projection_dyn_filter_case ( ProjectionDynFilterTestCase {
1911+ schema : Arc :: clone ( & schema) ,
1912+ batches : simple_abc. clone ( ) ,
1913+ projection : vec ! [
1914+ ( col( "b" , & schema) . unwrap( ) , "b" . to_string( ) ) ,
1915+ ( col( "a" , & schema) . unwrap( ) , "a" . to_string( ) ) ,
1916+ ] ,
1917+ sort_expr : PhysicalSortExpr :: new (
1918+ Arc :: new ( Column :: new ( "a" , 1 ) ) ,
1919+ SortOptions :: default ( ) ,
1920+ ) ,
1921+ expected_plans : vec ! [
1922+ r#" - SortExec: TopK(fetch=2), expr=[a@1 ASC], preserve_partitioning=[false]
1923+ - ProjectionExec: expr=[b@1 as b, a@0 as a]
1924+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"# . to_string( ) ,
1925+ r#" - SortExec: TopK(fetch=2), expr=[a@1 ASC], preserve_partitioning=[false], filter=[a@1 IS NULL OR a@1 < 2]
1926+ - ProjectionExec: expr=[b@1 as b, a@0 as a]
1927+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"# . to_string( ) ]
1928+ } )
1929+ . await ;
1930+
1931+ // Case 2: Pruning [a]
1932+ run_projection_dyn_filter_case ( ProjectionDynFilterTestCase {
1933+ schema : Arc :: clone ( & schema) ,
1934+ batches : simple_abc. clone ( ) ,
1935+ projection : vec ! [ ( col( "a" , & schema) . unwrap( ) , "a" . to_string( ) ) ] ,
1936+ sort_expr : PhysicalSortExpr :: new (
1937+ Arc :: new ( Column :: new ( "a" , 0 ) ) ,
1938+ SortOptions :: default ( ) ,
1939+ ) ,
1940+ expected_plans : vec ! [
1941+ r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false]
1942+ - ProjectionExec: expr=[a@0 as a]
1943+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"# . to_string( ) ,
1944+ r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false], filter=[a@0 IS NULL OR a@0 < 2]
1945+ - ProjectionExec: expr=[a@0 as a]
1946+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"# . to_string( ) ,
1947+ ] ,
1948+ } )
1949+ . await ;
1950+
1951+ // Case 3: Identity [a, b]
1952+ run_projection_dyn_filter_case ( ProjectionDynFilterTestCase {
1953+ schema : Arc :: clone ( & schema) ,
1954+ batches : simple_abc. clone ( ) ,
1955+ projection : vec ! [
1956+ ( col( "a" , & schema) . unwrap( ) , "a" . to_string( ) ) ,
1957+ ( col( "b" , & schema) . unwrap( ) , "b" . to_string( ) ) ,
1958+ ] ,
1959+ sort_expr : PhysicalSortExpr :: new (
1960+ Arc :: new ( Column :: new ( "a" , 0 ) ) ,
1961+ SortOptions :: default ( ) ,
1962+ ) ,
1963+ expected_plans : vec ! [
1964+ r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false]
1965+ - ProjectionExec: expr=[a@0 as a, b@1 as b]
1966+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"# . to_string( ) ,
1967+ r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false], filter=[a@0 IS NULL OR a@0 < 2]
1968+ - ProjectionExec: expr=[a@0 as a, b@1 as b]
1969+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"# . to_string( ) ,
1970+ ] ,
1971+ } )
1972+ . await ;
1973+
1974+ // Case 4: Expressions [a + 1, b]
1975+ run_projection_dyn_filter_case ( ProjectionDynFilterTestCase {
1976+ schema : Arc :: clone ( & schema) ,
1977+ batches : simple_abc. clone ( ) ,
1978+ projection : vec ! [
1979+ (
1980+ Arc :: new( BinaryExpr :: new(
1981+ col( "a" , & schema) . unwrap( ) ,
1982+ Operator :: Plus ,
1983+ Arc :: new( Literal :: new( ScalarValue :: Int32 ( Some ( 1 ) ) ) ) ,
1984+ ) ) ,
1985+ "a_plus_1" . to_string( ) ,
1986+ ) ,
1987+ ( col( "b" , & schema) . unwrap( ) , "b" . to_string( ) ) ,
1988+ ] ,
1989+ sort_expr : PhysicalSortExpr :: new (
1990+ Arc :: new ( Column :: new ( "a_plus_1" , 0 ) ) ,
1991+ SortOptions :: default ( ) ,
1992+ ) ,
1993+ expected_plans : vec ! [
1994+ r#" - SortExec: TopK(fetch=2), expr=[a_plus_1@0 ASC], preserve_partitioning=[false]
1995+ - ProjectionExec: expr=[a@0 + 1 as a_plus_1, b@1 as b]
1996+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"# . to_string( ) ,
1997+ r#" - SortExec: TopK(fetch=2), expr=[a_plus_1@0 ASC], preserve_partitioning=[false], filter=[a_plus_1@0 IS NULL OR a_plus_1@0 < 3]
1998+ - ProjectionExec: expr=[a@0 + 1 as a_plus_1, b@1 as b]
1999+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 + 1 IS NULL OR a@0 + 1 < 3 ]"# . to_string( ) ,
2000+ ] ,
2001+ } )
2002+ . await ;
2003+
2004+ // Case 5: [a as b, b as a] (swapped columns)
2005+ run_projection_dyn_filter_case ( ProjectionDynFilterTestCase {
2006+ schema : Arc :: clone ( & schema) ,
2007+ batches : simple_abc. clone ( ) ,
2008+ projection : vec ! [
2009+ ( col( "a" , & schema) . unwrap( ) , "b" . to_string( ) ) ,
2010+ ( col( "b" , & schema) . unwrap( ) , "a" . to_string( ) ) ,
2011+ ] ,
2012+ sort_expr : PhysicalSortExpr :: new (
2013+ Arc :: new ( Column :: new ( "b" , 0 ) ) ,
2014+ SortOptions :: default ( ) ,
2015+ ) ,
2016+ expected_plans : vec ! [
2017+ r#" - SortExec: TopK(fetch=2), expr=[b@0 ASC], preserve_partitioning=[false]
2018+ - ProjectionExec: expr=[a@0 as b, b@1 as a]
2019+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"# . to_string( ) ,
2020+ r#" - SortExec: TopK(fetch=2), expr=[b@0 ASC], preserve_partitioning=[false], filter=[b@0 IS NULL OR b@0 < 2]
2021+ - ProjectionExec: expr=[a@0 as b, b@1 as a]
2022+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"# . to_string( ) ,
2023+ ] ,
2024+ } )
2025+ . await ;
2026+
2027+ // Case 6: Confusing expr [a + 1 as a, b]
2028+ run_projection_dyn_filter_case ( ProjectionDynFilterTestCase {
2029+ schema : Arc :: clone ( & schema) ,
2030+ batches : simple_abc. clone ( ) ,
2031+ projection : vec ! [
2032+ (
2033+ Arc :: new( BinaryExpr :: new(
2034+ col( "a" , & schema) . unwrap( ) ,
2035+ Operator :: Plus ,
2036+ Arc :: new( Literal :: new( ScalarValue :: Int32 ( Some ( 1 ) ) ) ) ,
2037+ ) ) ,
2038+ "a" . to_string( ) ,
2039+ ) ,
2040+ ( col( "b" , & schema) . unwrap( ) , "b" . to_string( ) ) ,
2041+ ] ,
2042+ sort_expr : PhysicalSortExpr :: new (
2043+ Arc :: new ( Column :: new ( "a" , 0 ) ) ,
2044+ SortOptions :: default ( ) ,
2045+ ) ,
2046+ expected_plans : vec ! [
2047+ r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false]
2048+ - ProjectionExec: expr=[a@0 + 1 as a, b@1 as b]
2049+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"# . to_string( ) ,
2050+ r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false], filter=[a@0 IS NULL OR a@0 < 3]
2051+ - ProjectionExec: expr=[a@0 + 1 as a, b@1 as b]
2052+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 + 1 IS NULL OR a@0 + 1 < 3 ]"# . to_string( ) ,
2053+ ] ,
2054+ } )
2055+ . await ;
2056+ }
2057+
18292058/// Returns a predicate that is a binary expression col = lit
18302059fn col_lit_predicate (
18312060 column_name : & str ,
0 commit comments