@@ -59,6 +59,7 @@ use datafusion_physical_plan::{
5959 coalesce_partitions:: CoalescePartitionsExec ,
6060 collect,
6161 filter:: FilterExec ,
62+ projection:: ProjectionExec ,
6263 repartition:: RepartitionExec ,
6364 sorts:: sort:: SortExec ,
6465} ;
@@ -1813,6 +1814,234 @@ fn schema() -> SchemaRef {
18131814 Arc :: clone ( & TEST_SCHEMA )
18141815}
18151816
1817+ struct ProjectionDynFilterTestCase {
1818+ schema : SchemaRef ,
1819+ batches : Vec < RecordBatch > ,
1820+ projection : Vec < ( Arc < dyn PhysicalExpr > , String ) > ,
1821+ sort_expr : PhysicalSortExpr ,
1822+ expected_plans : Vec < String > ,
1823+ }
1824+
1825+ async fn run_projection_dyn_filter_case ( case : ProjectionDynFilterTestCase ) {
1826+ let ProjectionDynFilterTestCase {
1827+ schema,
1828+ batches,
1829+ projection,
1830+ sort_expr,
1831+ expected_plans,
1832+ } = case;
1833+
1834+ let scan = TestScanBuilder :: new ( Arc :: clone ( & schema) )
1835+ . with_support ( true )
1836+ . with_batches ( batches)
1837+ . build ( ) ;
1838+
1839+ let projection_exec = Arc :: new ( ProjectionExec :: try_new ( projection, scan) . unwrap ( ) ) ;
1840+
1841+ let sort = Arc :: new (
1842+ SortExec :: new ( LexOrdering :: new ( vec ! [ sort_expr] ) . unwrap ( ) , projection_exec)
1843+ . with_fetch ( Some ( 2 ) ) ,
1844+ ) as Arc < dyn ExecutionPlan > ;
1845+
1846+ let mut config = ConfigOptions :: default ( ) ;
1847+ config. execution . parquet . pushdown_filters = true ;
1848+ config. optimizer . enable_dynamic_filter_pushdown = true ;
1849+
1850+ let optimized_plan = FilterPushdown :: new_post_optimization ( )
1851+ . optimize ( Arc :: clone ( & sort) , & config)
1852+ . unwrap ( ) ;
1853+
1854+ pretty_assertions:: assert_eq!(
1855+ format_plan_for_test( & optimized_plan) . trim( ) ,
1856+ expected_plans[ 0 ] . trim( )
1857+ ) ;
1858+
1859+ let config = SessionConfig :: new ( ) . with_batch_size ( 2 ) ;
1860+ let session_ctx = SessionContext :: new_with_config ( config) ;
1861+ session_ctx. register_object_store (
1862+ ObjectStoreUrl :: parse ( "test://" ) . unwrap ( ) . as_ref ( ) ,
1863+ Arc :: new ( InMemory :: new ( ) ) ,
1864+ ) ;
1865+ let state = session_ctx. state ( ) ;
1866+ let task_ctx = state. task_ctx ( ) ;
1867+ let mut stream = optimized_plan. execute ( 0 , Arc :: clone ( & task_ctx) ) . unwrap ( ) ;
1868+ for ( idx, expected_plan) in expected_plans. iter ( ) . enumerate ( ) . skip ( 1 ) {
1869+ stream. next ( ) . await . unwrap ( ) . unwrap ( ) ;
1870+ let formatted_plan = format_plan_for_test ( & optimized_plan) ;
1871+ pretty_assertions:: assert_eq!(
1872+ formatted_plan. trim( ) ,
1873+ expected_plan. trim( ) ,
1874+ "Mismatch at iteration {}" ,
1875+ idx
1876+ ) ;
1877+ }
1878+ }
1879+
1880+ #[ tokio:: test]
1881+ async fn test_topk_with_projection_transformation_on_dyn_filter ( ) {
1882+ let schema = Arc :: new ( Schema :: new ( vec ! [
1883+ Field :: new( "a" , DataType :: Int32 , false ) ,
1884+ Field :: new( "b" , DataType :: Utf8 , false ) ,
1885+ Field :: new( "c" , DataType :: Float64 , false ) ,
1886+ ] ) ) ;
1887+ let simple_abc = vec ! [
1888+ record_batch!(
1889+ ( "a" , Int32 , [ 1 , 2 , 3 ] ) ,
1890+ ( "b" , Utf8 , [ "x" , "y" , "z" ] ) ,
1891+ ( "c" , Float64 , [ 1.0 , 2.0 , 3.0 ] )
1892+ )
1893+ . unwrap( ) ,
1894+ ] ;
1895+
1896+ // Case 1: Reordering [b, a]
1897+ run_projection_dyn_filter_case ( ProjectionDynFilterTestCase {
1898+ schema : Arc :: clone ( & schema) ,
1899+ batches : simple_abc. clone ( ) ,
1900+ projection : vec ! [
1901+ ( col( "b" , & schema) . unwrap( ) , "b" . to_string( ) ) ,
1902+ ( col( "a" , & schema) . unwrap( ) , "a" . to_string( ) ) ,
1903+ ] ,
1904+ sort_expr : PhysicalSortExpr :: new (
1905+ Arc :: new ( Column :: new ( "a" , 1 ) ) ,
1906+ SortOptions :: default ( ) ,
1907+ ) ,
1908+ expected_plans : vec ! [
1909+ r#" - SortExec: TopK(fetch=2), expr=[a@1 ASC], preserve_partitioning=[false]
1910+ - ProjectionExec: expr=[b@1 as b, a@0 as a]
1911+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"# . to_string( ) ,
1912+ r#" - SortExec: TopK(fetch=2), expr=[a@1 ASC], preserve_partitioning=[false], filter=[a@1 IS NULL OR a@1 < 2]
1913+ - ProjectionExec: expr=[b@1 as b, a@0 as a]
1914+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"# . to_string( ) ]
1915+ } )
1916+ . await ;
1917+
1918+ // Case 2: Pruning [a]
1919+ run_projection_dyn_filter_case ( ProjectionDynFilterTestCase {
1920+ schema : Arc :: clone ( & schema) ,
1921+ batches : simple_abc. clone ( ) ,
1922+ projection : vec ! [ ( col( "a" , & schema) . unwrap( ) , "a" . to_string( ) ) ] ,
1923+ sort_expr : PhysicalSortExpr :: new (
1924+ Arc :: new ( Column :: new ( "a" , 0 ) ) ,
1925+ SortOptions :: default ( ) ,
1926+ ) ,
1927+ expected_plans : vec ! [
1928+ r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false]
1929+ - ProjectionExec: expr=[a@0 as a]
1930+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"# . to_string( ) ,
1931+ r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false], filter=[a@0 IS NULL OR a@0 < 2]
1932+ - ProjectionExec: expr=[a@0 as a]
1933+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"# . to_string( ) ,
1934+ ] ,
1935+ } )
1936+ . await ;
1937+
1938+ // Case 3: Identity [a, b]
1939+ run_projection_dyn_filter_case ( ProjectionDynFilterTestCase {
1940+ schema : Arc :: clone ( & schema) ,
1941+ batches : simple_abc. clone ( ) ,
1942+ projection : vec ! [
1943+ ( col( "a" , & schema) . unwrap( ) , "a" . to_string( ) ) ,
1944+ ( col( "b" , & schema) . unwrap( ) , "b" . to_string( ) ) ,
1945+ ] ,
1946+ sort_expr : PhysicalSortExpr :: new (
1947+ Arc :: new ( Column :: new ( "a" , 0 ) ) ,
1948+ SortOptions :: default ( ) ,
1949+ ) ,
1950+ expected_plans : vec ! [
1951+ r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false]
1952+ - ProjectionExec: expr=[a@0 as a, b@1 as b]
1953+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"# . to_string( ) ,
1954+ r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false], filter=[a@0 IS NULL OR a@0 < 2]
1955+ - ProjectionExec: expr=[a@0 as a, b@1 as b]
1956+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"# . to_string( ) ,
1957+ ] ,
1958+ } )
1959+ . await ;
1960+
1961+ // Case 4: Expressions [a + 1, b]
1962+ run_projection_dyn_filter_case ( ProjectionDynFilterTestCase {
1963+ schema : Arc :: clone ( & schema) ,
1964+ batches : simple_abc. clone ( ) ,
1965+ projection : vec ! [
1966+ (
1967+ Arc :: new( BinaryExpr :: new(
1968+ col( "a" , & schema) . unwrap( ) ,
1969+ Operator :: Plus ,
1970+ Arc :: new( Literal :: new( ScalarValue :: Int32 ( Some ( 1 ) ) ) ) ,
1971+ ) ) ,
1972+ "a_plus_1" . to_string( ) ,
1973+ ) ,
1974+ ( col( "b" , & schema) . unwrap( ) , "b" . to_string( ) ) ,
1975+ ] ,
1976+ sort_expr : PhysicalSortExpr :: new (
1977+ Arc :: new ( Column :: new ( "a_plus_1" , 0 ) ) ,
1978+ SortOptions :: default ( ) ,
1979+ ) ,
1980+ expected_plans : vec ! [
1981+ r#" - SortExec: TopK(fetch=2), expr=[a_plus_1@0 ASC], preserve_partitioning=[false]
1982+ - ProjectionExec: expr=[a@0 + 1 as a_plus_1, b@1 as b]
1983+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"# . to_string( ) ,
1984+ r#" - SortExec: TopK(fetch=2), expr=[a_plus_1@0 ASC], preserve_partitioning=[false], filter=[a_plus_1@0 IS NULL OR a_plus_1@0 < 3]
1985+ - ProjectionExec: expr=[a@0 + 1 as a_plus_1, b@1 as b]
1986+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 + 1 IS NULL OR a@0 + 1 < 3 ]"# . to_string( ) ,
1987+ ] ,
1988+ } )
1989+ . await ;
1990+
1991+ // Case 5: [a as b, b as a] (swapped columns)
1992+ run_projection_dyn_filter_case ( ProjectionDynFilterTestCase {
1993+ schema : Arc :: clone ( & schema) ,
1994+ batches : simple_abc. clone ( ) ,
1995+ projection : vec ! [
1996+ ( col( "a" , & schema) . unwrap( ) , "b" . to_string( ) ) ,
1997+ ( col( "b" , & schema) . unwrap( ) , "a" . to_string( ) ) ,
1998+ ] ,
1999+ sort_expr : PhysicalSortExpr :: new (
2000+ Arc :: new ( Column :: new ( "b" , 0 ) ) ,
2001+ SortOptions :: default ( ) ,
2002+ ) ,
2003+ expected_plans : vec ! [
2004+ r#" - SortExec: TopK(fetch=2), expr=[b@0 ASC], preserve_partitioning=[false]
2005+ - ProjectionExec: expr=[a@0 as b, b@1 as a]
2006+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"# . to_string( ) ,
2007+ r#" - SortExec: TopK(fetch=2), expr=[b@0 ASC], preserve_partitioning=[false], filter=[b@0 IS NULL OR b@0 < 2]
2008+ - ProjectionExec: expr=[a@0 as b, b@1 as a]
2009+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"# . to_string( ) ,
2010+ ] ,
2011+ } )
2012+ . await ;
2013+
2014+ // Case 6: Confusing expr [a + 1 as a, b]
2015+ run_projection_dyn_filter_case ( ProjectionDynFilterTestCase {
2016+ schema : Arc :: clone ( & schema) ,
2017+ batches : simple_abc. clone ( ) ,
2018+ projection : vec ! [
2019+ (
2020+ Arc :: new( BinaryExpr :: new(
2021+ col( "a" , & schema) . unwrap( ) ,
2022+ Operator :: Plus ,
2023+ Arc :: new( Literal :: new( ScalarValue :: Int32 ( Some ( 1 ) ) ) ) ,
2024+ ) ) ,
2025+ "a" . to_string( ) ,
2026+ ) ,
2027+ ( col( "b" , & schema) . unwrap( ) , "b" . to_string( ) ) ,
2028+ ] ,
2029+ sort_expr : PhysicalSortExpr :: new (
2030+ Arc :: new ( Column :: new ( "a" , 0 ) ) ,
2031+ SortOptions :: default ( ) ,
2032+ ) ,
2033+ expected_plans : vec ! [
2034+ r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false]
2035+ - ProjectionExec: expr=[a@0 + 1 as a, b@1 as b]
2036+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"# . to_string( ) ,
2037+ r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false], filter=[a@0 IS NULL OR a@0 < 3]
2038+ - ProjectionExec: expr=[a@0 + 1 as a, b@1 as b]
2039+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 + 1 IS NULL OR a@0 + 1 < 3 ]"# . to_string( ) ,
2040+ ] ,
2041+ } )
2042+ . await ;
2043+ }
2044+
18162045/// Returns a predicate that is a binary expression col = lit
18172046fn col_lit_predicate (
18182047 column_name : & str ,
0 commit comments