@@ -34,6 +34,7 @@ use datafusion::{
3434 scalar:: ScalarValue ,
3535} ;
3636use datafusion_catalog:: memory:: DataSourceExec ;
37+ use datafusion_common:: JoinType ;
3738use datafusion_common:: config:: ConfigOptions ;
3839use datafusion_datasource:: {
3940 PartitionedFile , file_groups:: FileGroup , file_scan_config:: FileScanConfigBuilder ,
@@ -53,6 +54,7 @@ use datafusion_physical_expr::{
5354use datafusion_physical_optimizer:: {
5455 PhysicalOptimizerRule , filter_pushdown:: FilterPushdown ,
5556} ;
57+ use datafusion_physical_plan:: joins:: { HashJoinExec , PartitionMode } ;
5658use datafusion_physical_plan:: {
5759 ExecutionPlan ,
5860 aggregates:: { AggregateExec , AggregateMode , PhysicalGroupBy } ,
@@ -3145,6 +3147,346 @@ fn test_pushdown_with_empty_group_by() {
31453147 ) ;
31463148}
31473149
3150+ #[ test]
3151+ fn test_pushdown_through_aggregate_with_reordered_input_columns ( ) {
3152+ let scan = TestScanBuilder :: new ( schema ( ) ) . with_support ( true ) . build ( ) ;
3153+
3154+ // Reorder scan output from (a, b, c) to (c, a, b)
3155+ let reordered_schema = Arc :: new ( Schema :: new ( vec ! [
3156+ Field :: new( "c" , DataType :: Float64 , false ) ,
3157+ Field :: new( "a" , DataType :: Utf8 , false ) ,
3158+ Field :: new( "b" , DataType :: Utf8 , false ) ,
3159+ ] ) ) ;
3160+ let projection = Arc :: new (
3161+ ProjectionExec :: try_new (
3162+ vec ! [
3163+ ( col( "c" , & schema( ) ) . unwrap( ) , "c" . to_string( ) ) ,
3164+ ( col( "a" , & schema( ) ) . unwrap( ) , "a" . to_string( ) ) ,
3165+ ( col( "b" , & schema( ) ) . unwrap( ) , "b" . to_string( ) ) ,
3166+ ] ,
3167+ scan,
3168+ )
3169+ . unwrap ( ) ,
3170+ ) ;
3171+
3172+ let aggregate_expr = vec ! [
3173+ AggregateExprBuilder :: new(
3174+ count_udaf( ) ,
3175+ vec![ col( "c" , & reordered_schema) . unwrap( ) ] ,
3176+ )
3177+ . schema( reordered_schema. clone( ) )
3178+ . alias( "cnt" )
3179+ . build( )
3180+ . map( Arc :: new)
3181+ . unwrap( ) ,
3182+ ] ;
3183+
3184+ // Group by a@1, b@2 (input indices in reordered schema)
3185+ let group_by = PhysicalGroupBy :: new_single ( vec ! [
3186+ ( col( "a" , & reordered_schema) . unwrap( ) , "a" . to_string( ) ) ,
3187+ ( col( "b" , & reordered_schema) . unwrap( ) , "b" . to_string( ) ) ,
3188+ ] ) ;
3189+
3190+ let aggregate = Arc :: new (
3191+ AggregateExec :: try_new (
3192+ AggregateMode :: Final ,
3193+ group_by,
3194+ aggregate_expr,
3195+ vec ! [ None ] ,
3196+ projection,
3197+ reordered_schema,
3198+ )
3199+ . unwrap ( ) ,
3200+ ) ;
3201+
3202+ // Filter on b@1 in aggregate's output schema (a@0, b@1, cnt@2)
3203+ // The grouping expr for b references input index 2, but output index is 1.
3204+ let agg_output_schema = aggregate. schema ( ) ;
3205+ let predicate = col_lit_predicate ( "b" , "bar" , & agg_output_schema) ;
3206+ let plan = Arc :: new ( FilterExec :: try_new ( predicate, aggregate) . unwrap ( ) ) ;
3207+
3208+ // The filter should be pushed down
3209+ insta:: assert_snapshot!(
3210+ OptimizationTest :: new( plan, FilterPushdown :: new( ) , true ) ,
3211+ @r"
3212+ OptimizationTest:
3213+ input:
3214+ - FilterExec: b@1 = bar
3215+ - AggregateExec: mode=Final, gby=[a@1 as a, b@2 as b], aggr=[cnt]
3216+ - ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]
3217+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
3218+ output:
3219+ Ok:
3220+ - AggregateExec: mode=Final, gby=[a@1 as a, b@2 as b], aggr=[cnt], ordering_mode=PartiallySorted([1])
3221+ - ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]
3222+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar
3223+ "
3224+ ) ;
3225+ }
3226+
3227+ #[ test]
3228+ fn test_pushdown_through_aggregate_grouping_sets_with_reordered_input ( ) {
3229+ let scan = TestScanBuilder :: new ( schema ( ) ) . with_support ( true ) . build ( ) ;
3230+
3231+ let reordered_schema = Arc :: new ( Schema :: new ( vec ! [
3232+ Field :: new( "c" , DataType :: Float64 , false ) ,
3233+ Field :: new( "a" , DataType :: Utf8 , false ) ,
3234+ Field :: new( "b" , DataType :: Utf8 , false ) ,
3235+ ] ) ) ;
3236+ let projection = Arc :: new (
3237+ ProjectionExec :: try_new (
3238+ vec ! [
3239+ ( col( "c" , & schema( ) ) . unwrap( ) , "c" . to_string( ) ) ,
3240+ ( col( "a" , & schema( ) ) . unwrap( ) , "a" . to_string( ) ) ,
3241+ ( col( "b" , & schema( ) ) . unwrap( ) , "b" . to_string( ) ) ,
3242+ ] ,
3243+ scan,
3244+ )
3245+ . unwrap ( ) ,
3246+ ) ;
3247+
3248+ let aggregate_expr = vec ! [
3249+ AggregateExprBuilder :: new(
3250+ count_udaf( ) ,
3251+ vec![ col( "c" , & reordered_schema) . unwrap( ) ] ,
3252+ )
3253+ . schema( reordered_schema. clone( ) )
3254+ . alias( "cnt" )
3255+ . build( )
3256+ . map( Arc :: new)
3257+ . unwrap( ) ,
3258+ ] ;
3259+
3260+ // Use grouping sets (a, b) and (b).
3261+ let group_by = PhysicalGroupBy :: new (
3262+ vec ! [
3263+ ( col( "a" , & reordered_schema) . unwrap( ) , "a" . to_string( ) ) ,
3264+ ( col( "b" , & reordered_schema) . unwrap( ) , "b" . to_string( ) ) ,
3265+ ] ,
3266+ vec ! [
3267+ (
3268+ Arc :: new( Literal :: new( ScalarValue :: Utf8 ( None ) ) ) ,
3269+ "a" . to_string( ) ,
3270+ ) ,
3271+ (
3272+ Arc :: new( Literal :: new( ScalarValue :: Utf8 ( None ) ) ) ,
3273+ "b" . to_string( ) ,
3274+ ) ,
3275+ ] ,
3276+ vec ! [ vec![ false , false ] , vec![ true , false ] ] ,
3277+ true ,
3278+ ) ;
3279+
3280+ let aggregate = Arc :: new (
3281+ AggregateExec :: try_new (
3282+ AggregateMode :: Final ,
3283+ group_by,
3284+ aggregate_expr,
3285+ vec ! [ None ] ,
3286+ projection,
3287+ reordered_schema,
3288+ )
3289+ . unwrap ( ) ,
3290+ ) ;
3291+
3292+ let agg_output_schema = aggregate. schema ( ) ;
3293+
3294+ // Filter on b (present in all grouping sets) should be pushed down
3295+ let predicate = col_lit_predicate ( "b" , "bar" , & agg_output_schema) ;
3296+ let plan = Arc :: new ( FilterExec :: try_new ( predicate, aggregate. clone ( ) ) . unwrap ( ) ) ;
3297+
3298+ insta:: assert_snapshot!(
3299+ OptimizationTest :: new( plan, FilterPushdown :: new( ) , true ) ,
3300+ @r"
3301+ OptimizationTest:
3302+ input:
3303+ - FilterExec: b@1 = bar
3304+ - AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt]
3305+ - ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]
3306+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
3307+ output:
3308+ Ok:
3309+ - AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt], ordering_mode=PartiallySorted([1])
3310+ - ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]
3311+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar
3312+ "
3313+ ) ;
3314+
3315+ // Filter on a (missing from second grouping set) should not be pushed down
3316+ let predicate = col_lit_predicate ( "a" , "foo" , & agg_output_schema) ;
3317+ let plan = Arc :: new ( FilterExec :: try_new ( predicate, aggregate) . unwrap ( ) ) ;
3318+
3319+ insta:: assert_snapshot!(
3320+ OptimizationTest :: new( plan, FilterPushdown :: new( ) , true ) ,
3321+ @r"
3322+ OptimizationTest:
3323+ input:
3324+ - FilterExec: a@0 = foo
3325+ - AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt]
3326+ - ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]
3327+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
3328+ output:
3329+ Ok:
3330+ - FilterExec: a@0 = foo
3331+ - AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt]
3332+ - ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]
3333+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
3334+ "
3335+ ) ;
3336+ }
3337+
3338+ /// Regression test for https://github.com/apache/datafusion/issues/21065.
3339+ ///
3340+ /// Given a plan similar to the following, ensure that the filter is pushed down
3341+ /// through an AggregateExec whose input columns are reordered by a ProjectionExec.
3342+ #[ tokio:: test]
3343+ async fn test_hashjoin_dynamic_filter_pushdown_through_aggregate_with_reordered_input ( ) {
3344+ // Build side
3345+ let build_batches = vec ! [ record_batch!( ( "a" , Utf8 , [ "h1" , "h2" ] ) ) . unwrap( ) ] ;
3346+ let build_schema =
3347+ Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Utf8 , false ) ] ) ) ;
3348+ let build_scan = TestScanBuilder :: new ( Arc :: clone ( & build_schema) )
3349+ . with_support ( true )
3350+ . with_batches ( build_batches)
3351+ . build ( ) ;
3352+
3353+ // Probe side
3354+ let probe_batches = vec ! [
3355+ record_batch!(
3356+ ( "a" , Utf8 , [ "h1" , "h2" , "h3" , "h4" ] ) ,
3357+ ( "value" , Float64 , [ 1.0 , 2.0 , 3.0 , 4.0 ] )
3358+ )
3359+ . unwrap( ) ,
3360+ ] ;
3361+ let probe_schema = Arc :: new ( Schema :: new ( vec ! [
3362+ Field :: new( "a" , DataType :: Utf8 , false ) ,
3363+ Field :: new( "value" , DataType :: Float64 , false ) ,
3364+ ] ) ) ;
3365+ let probe_scan = TestScanBuilder :: new ( Arc :: clone ( & probe_schema) )
3366+ . with_support ( true )
3367+ . with_batches ( probe_batches)
3368+ . build ( ) ;
3369+
3370+ // ProjectionExec reorders (a, value) → (value, a)
3371+ let reordered_schema = Arc :: new ( Schema :: new ( vec ! [
3372+ Field :: new( "value" , DataType :: Float64 , false ) ,
3373+ Field :: new( "a" , DataType :: Utf8 , false ) ,
3374+ ] ) ) ;
3375+ let projection = Arc :: new (
3376+ ProjectionExec :: try_new (
3377+ vec ! [
3378+ ( col( "value" , & probe_schema) . unwrap( ) , "value" . to_string( ) ) ,
3379+ ( col( "a" , & probe_schema) . unwrap( ) , "a" . to_string( ) ) ,
3380+ ] ,
3381+ probe_scan,
3382+ )
3383+ . unwrap ( ) ,
3384+ ) ;
3385+
3386+ // AggregateExec: GROUP BY a@1, min(value@0)
3387+ let aggregate_expr = vec ! [
3388+ AggregateExprBuilder :: new(
3389+ min_udaf( ) ,
3390+ vec![ col( "value" , & reordered_schema) . unwrap( ) ] ,
3391+ )
3392+ . schema( reordered_schema. clone( ) )
3393+ . alias( "min_value" )
3394+ . build( )
3395+ . map( Arc :: new)
3396+ . unwrap( ) ,
3397+ ] ;
3398+ let group_by = PhysicalGroupBy :: new_single ( vec ! [ (
3399+ col( "a" , & reordered_schema) . unwrap( ) , // a@1 in input
3400+ "a" . to_string( ) ,
3401+ ) ] ) ;
3402+
3403+ let aggregate = Arc :: new (
3404+ AggregateExec :: try_new (
3405+ AggregateMode :: Single ,
3406+ group_by,
3407+ aggregate_expr,
3408+ vec ! [ None ] ,
3409+ projection,
3410+ reordered_schema,
3411+ )
3412+ . unwrap ( ) ,
3413+ ) ;
3414+
3415+ // Aggregate output schema: (a@0, min_value@1)
3416+ let agg_output_schema = aggregate. schema ( ) ;
3417+
3418+ // Join the build and probe side
3419+ let plan = Arc :: new (
3420+ HashJoinExec :: try_new (
3421+ build_scan,
3422+ aggregate,
3423+ vec ! [ (
3424+ col( "a" , & build_schema) . unwrap( ) ,
3425+ col( "a" , & agg_output_schema) . unwrap( ) ,
3426+ ) ] ,
3427+ None ,
3428+ & JoinType :: Inner ,
3429+ None ,
3430+ PartitionMode :: CollectLeft ,
3431+ datafusion_common:: NullEquality :: NullEqualsNothing ,
3432+ false ,
3433+ )
3434+ . unwrap ( ) ,
3435+ ) as Arc < dyn ExecutionPlan > ;
3436+
3437+ // The HashJoin's dynamic filter on `a` should push
3438+ // through the aggregate and reach the probe-side DataSource.
3439+ insta:: assert_snapshot!(
3440+ OptimizationTest :: new( Arc :: clone( & plan) , FilterPushdown :: new_post_optimization( ) , true ) ,
3441+ @r"
3442+ OptimizationTest:
3443+ input:
3444+ - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0)]
3445+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a], file_type=test, pushdown_supported=true
3446+ - AggregateExec: mode=Single, gby=[a@1 as a], aggr=[min_value]
3447+ - ProjectionExec: expr=[value@1 as value, a@0 as a]
3448+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, value], file_type=test, pushdown_supported=true
3449+ output:
3450+ Ok:
3451+ - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0)]
3452+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a], file_type=test, pushdown_supported=true
3453+ - AggregateExec: mode=Single, gby=[a@1 as a], aggr=[min_value]
3454+ - ProjectionExec: expr=[value@1 as value, a@0 as a]
3455+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, value], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]
3456+ "
3457+ ) ;
3458+
3459+ // Actually execute the plan to verify the dynamic filter is populated
3460+ let mut config = ConfigOptions :: default ( ) ;
3461+ config. execution . parquet . pushdown_filters = true ;
3462+ let plan = FilterPushdown :: new_post_optimization ( )
3463+ . optimize ( plan, & config)
3464+ . unwrap ( ) ;
3465+
3466+ let session_config = SessionConfig :: new ( ) . with_batch_size ( 10 ) ;
3467+ let session_ctx = SessionContext :: new_with_config ( session_config) ;
3468+ session_ctx. register_object_store (
3469+ ObjectStoreUrl :: parse ( "test://" ) . unwrap ( ) . as_ref ( ) ,
3470+ Arc :: new ( InMemory :: new ( ) ) ,
3471+ ) ;
3472+ let state = session_ctx. state ( ) ;
3473+ let task_ctx = state. task_ctx ( ) ;
3474+ let mut stream = plan. execute ( 0 , Arc :: clone ( & task_ctx) ) . unwrap ( ) ;
3475+ stream. next ( ) . await . unwrap ( ) . unwrap ( ) ;
3476+
3477+ // After execution, the dynamic filter should be populated with values
3478+ insta:: assert_snapshot!(
3479+ format!( "{}" , format_plan_for_test( & plan) ) ,
3480+ @r"
3481+ - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0)]
3482+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a], file_type=test, pushdown_supported=true
3483+ - AggregateExec: mode=Single, gby=[a@1 as a], aggr=[min_value]
3484+ - ProjectionExec: expr=[value@1 as value, a@0 as a]
3485+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, value], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 >= h1 AND a@0 <= h2 AND a@0 IN (SET) ([h1, h2]) ]
3486+ "
3487+ ) ;
3488+ }
3489+
31483490#[ test]
31493491fn test_pushdown_with_computed_grouping_key ( ) {
31503492 // Test filter pushdown with computed grouping expression
0 commit comments