Skip to content

Commit 114139c

Browse files
ensure dynamic filters are correctly pushed down through aggregations (apache#21059)
- Closes apache#21065. In plans such as the following, dynamic filters are not pushed down through the aggregation ``` CREATE TABLE data (a VARCHAR, ts TIMESTAMP, value DOUBLE) AS VALUES ('h1', '2024-01-01T00:05:00', 1.0), ('h1', '2024-01-01T00:15:00', 2.0), ('h2', '2024-01-01T00:25:00', 3.0), ('h3', '2024-01-01T00:35:00', 4.0); SELECT * FROM contexts c INNER JOIN ( SELECT a, date_bin(interval '1 hour', ts) AS bucket, min(value) AS min_val FROM (SELECT value, a, ts FROM data) GROUP BY a, date_bin(interval '1 hour', ts) ) agg ON c.a = agg.a; ``` ``` HashJoinExec: mode=Auto, join_type=Inner, on=[(a@0, a@0)] DataSourceExec: partitions=1 ProjectionExec: [a@0, date_bin(1h, ts)@1 as bucket, min(value)@2 as min_val] AggregateExec: mode=FinalPartitioned, gby=[a@0, date_bin(1h, ts)@1], aggr=[min(value)] AggregateExec: mode=Partial, gby=[a@1, date_bin(1h, ts@2)], aggr=[min(value)] ProjectionExec: [value@2, a@0, ts@1] ← reorders columns DataSourceExec: partitions=1 ``` `AggregateExec::gather_filters_for_pushdown` compared parent filter columns (output schema indices) against grouping expression columns (input schema indices). When a `ProjectionExec` below the aggregate reorders columns, the index mismatch causes filters (such as HashJoin dynamic filters) to be incorrectly blocked. This change fixes the column index mapping in `AggregateExec::gather_filters_for_pushdown` - `test_pushdown_through_aggregate_with_reordered_input_columns` — filter on grouping column with reordered input is pushed down - `test_pushdown_through_aggregate_with_reordered_input_no_pushdown_on_agg_result` — filter on aggregate result column is not pushed down - `test_pushdown_through_aggregate_grouping_sets_with_reordered_input` — GROUPING SETS: filter on common column pushed, filter on missing column blocked - `test_hashjoin_dynamic_filter_pushdown_through_aggregate_with_reordered_input` — HashJoin dynamic filter pushes through aggregate with reordered input and is populated with values after execution - All tests verified to fail without the fix No.
1 parent aca07c4 commit 114139c

3 files changed

Lines changed: 417 additions & 9 deletions

File tree

‎datafusion/core/tests/physical_optimizer/filter_pushdown.rs‎

Lines changed: 342 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ use datafusion::{
3434
scalar::ScalarValue,
3535
};
3636
use datafusion_catalog::memory::DataSourceExec;
37+
use datafusion_common::JoinType;
3738
use datafusion_common::config::ConfigOptions;
3839
use datafusion_datasource::{
3940
PartitionedFile, file_groups::FileGroup, file_scan_config::FileScanConfigBuilder,
@@ -53,6 +54,7 @@ use datafusion_physical_expr::{
5354
use datafusion_physical_optimizer::{
5455
PhysicalOptimizerRule, filter_pushdown::FilterPushdown,
5556
};
57+
use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};
5658
use datafusion_physical_plan::{
5759
ExecutionPlan,
5860
aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy},
@@ -3145,6 +3147,346 @@ fn test_pushdown_with_empty_group_by() {
31453147
);
31463148
}
31473149

3150+
#[test]
3151+
fn test_pushdown_through_aggregate_with_reordered_input_columns() {
3152+
let scan = TestScanBuilder::new(schema()).with_support(true).build();
3153+
3154+
// Reorder scan output from (a, b, c) to (c, a, b)
3155+
let reordered_schema = Arc::new(Schema::new(vec![
3156+
Field::new("c", DataType::Float64, false),
3157+
Field::new("a", DataType::Utf8, false),
3158+
Field::new("b", DataType::Utf8, false),
3159+
]));
3160+
let projection = Arc::new(
3161+
ProjectionExec::try_new(
3162+
vec![
3163+
(col("c", &schema()).unwrap(), "c".to_string()),
3164+
(col("a", &schema()).unwrap(), "a".to_string()),
3165+
(col("b", &schema()).unwrap(), "b".to_string()),
3166+
],
3167+
scan,
3168+
)
3169+
.unwrap(),
3170+
);
3171+
3172+
let aggregate_expr = vec![
3173+
AggregateExprBuilder::new(
3174+
count_udaf(),
3175+
vec![col("c", &reordered_schema).unwrap()],
3176+
)
3177+
.schema(reordered_schema.clone())
3178+
.alias("cnt")
3179+
.build()
3180+
.map(Arc::new)
3181+
.unwrap(),
3182+
];
3183+
3184+
// Group by a@1, b@2 (input indices in reordered schema)
3185+
let group_by = PhysicalGroupBy::new_single(vec![
3186+
(col("a", &reordered_schema).unwrap(), "a".to_string()),
3187+
(col("b", &reordered_schema).unwrap(), "b".to_string()),
3188+
]);
3189+
3190+
let aggregate = Arc::new(
3191+
AggregateExec::try_new(
3192+
AggregateMode::Final,
3193+
group_by,
3194+
aggregate_expr,
3195+
vec![None],
3196+
projection,
3197+
reordered_schema,
3198+
)
3199+
.unwrap(),
3200+
);
3201+
3202+
// Filter on b@1 in aggregate's output schema (a@0, b@1, cnt@2)
3203+
// The grouping expr for b references input index 2, but output index is 1.
3204+
let agg_output_schema = aggregate.schema();
3205+
let predicate = col_lit_predicate("b", "bar", &agg_output_schema);
3206+
let plan = Arc::new(FilterExec::try_new(predicate, aggregate).unwrap());
3207+
3208+
// The filter should be pushed down
3209+
insta::assert_snapshot!(
3210+
OptimizationTest::new(plan, FilterPushdown::new(), true),
3211+
@r"
3212+
OptimizationTest:
3213+
input:
3214+
- FilterExec: b@1 = bar
3215+
- AggregateExec: mode=Final, gby=[a@1 as a, b@2 as b], aggr=[cnt]
3216+
- ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]
3217+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
3218+
output:
3219+
Ok:
3220+
- AggregateExec: mode=Final, gby=[a@1 as a, b@2 as b], aggr=[cnt], ordering_mode=PartiallySorted([1])
3221+
- ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]
3222+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar
3223+
"
3224+
);
3225+
}
3226+
3227+
#[test]
3228+
fn test_pushdown_through_aggregate_grouping_sets_with_reordered_input() {
3229+
let scan = TestScanBuilder::new(schema()).with_support(true).build();
3230+
3231+
let reordered_schema = Arc::new(Schema::new(vec![
3232+
Field::new("c", DataType::Float64, false),
3233+
Field::new("a", DataType::Utf8, false),
3234+
Field::new("b", DataType::Utf8, false),
3235+
]));
3236+
let projection = Arc::new(
3237+
ProjectionExec::try_new(
3238+
vec![
3239+
(col("c", &schema()).unwrap(), "c".to_string()),
3240+
(col("a", &schema()).unwrap(), "a".to_string()),
3241+
(col("b", &schema()).unwrap(), "b".to_string()),
3242+
],
3243+
scan,
3244+
)
3245+
.unwrap(),
3246+
);
3247+
3248+
let aggregate_expr = vec![
3249+
AggregateExprBuilder::new(
3250+
count_udaf(),
3251+
vec![col("c", &reordered_schema).unwrap()],
3252+
)
3253+
.schema(reordered_schema.clone())
3254+
.alias("cnt")
3255+
.build()
3256+
.map(Arc::new)
3257+
.unwrap(),
3258+
];
3259+
3260+
// Use grouping sets (a, b) and (b).
3261+
let group_by = PhysicalGroupBy::new(
3262+
vec![
3263+
(col("a", &reordered_schema).unwrap(), "a".to_string()),
3264+
(col("b", &reordered_schema).unwrap(), "b".to_string()),
3265+
],
3266+
vec![
3267+
(
3268+
Arc::new(Literal::new(ScalarValue::Utf8(None))),
3269+
"a".to_string(),
3270+
),
3271+
(
3272+
Arc::new(Literal::new(ScalarValue::Utf8(None))),
3273+
"b".to_string(),
3274+
),
3275+
],
3276+
vec![vec![false, false], vec![true, false]],
3277+
true,
3278+
);
3279+
3280+
let aggregate = Arc::new(
3281+
AggregateExec::try_new(
3282+
AggregateMode::Final,
3283+
group_by,
3284+
aggregate_expr,
3285+
vec![None],
3286+
projection,
3287+
reordered_schema,
3288+
)
3289+
.unwrap(),
3290+
);
3291+
3292+
let agg_output_schema = aggregate.schema();
3293+
3294+
// Filter on b (present in all grouping sets) should be pushed down
3295+
let predicate = col_lit_predicate("b", "bar", &agg_output_schema);
3296+
let plan = Arc::new(FilterExec::try_new(predicate, aggregate.clone()).unwrap());
3297+
3298+
insta::assert_snapshot!(
3299+
OptimizationTest::new(plan, FilterPushdown::new(), true),
3300+
@r"
3301+
OptimizationTest:
3302+
input:
3303+
- FilterExec: b@1 = bar
3304+
- AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt]
3305+
- ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]
3306+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
3307+
output:
3308+
Ok:
3309+
- AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt], ordering_mode=PartiallySorted([1])
3310+
- ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]
3311+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar
3312+
"
3313+
);
3314+
3315+
// Filter on a (missing from second grouping set) should not be pushed down
3316+
let predicate = col_lit_predicate("a", "foo", &agg_output_schema);
3317+
let plan = Arc::new(FilterExec::try_new(predicate, aggregate).unwrap());
3318+
3319+
insta::assert_snapshot!(
3320+
OptimizationTest::new(plan, FilterPushdown::new(), true),
3321+
@r"
3322+
OptimizationTest:
3323+
input:
3324+
- FilterExec: a@0 = foo
3325+
- AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt]
3326+
- ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]
3327+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
3328+
output:
3329+
Ok:
3330+
- FilterExec: a@0 = foo
3331+
- AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt]
3332+
- ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]
3333+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
3334+
"
3335+
);
3336+
}
3337+
3338+
/// Regression test for https://github.com/apache/datafusion/issues/21065.
3339+
///
3340+
/// Given a plan similar to the following, ensure that the filter is pushed down
3341+
/// through an AggregateExec whose input columns are reordered by a ProjectionExec.
3342+
#[tokio::test]
3343+
async fn test_hashjoin_dynamic_filter_pushdown_through_aggregate_with_reordered_input() {
3344+
// Build side
3345+
let build_batches = vec![record_batch!(("a", Utf8, ["h1", "h2"])).unwrap()];
3346+
let build_schema =
3347+
Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false)]));
3348+
let build_scan = TestScanBuilder::new(Arc::clone(&build_schema))
3349+
.with_support(true)
3350+
.with_batches(build_batches)
3351+
.build();
3352+
3353+
// Probe side
3354+
let probe_batches = vec![
3355+
record_batch!(
3356+
("a", Utf8, ["h1", "h2", "h3", "h4"]),
3357+
("value", Float64, [1.0, 2.0, 3.0, 4.0])
3358+
)
3359+
.unwrap(),
3360+
];
3361+
let probe_schema = Arc::new(Schema::new(vec![
3362+
Field::new("a", DataType::Utf8, false),
3363+
Field::new("value", DataType::Float64, false),
3364+
]));
3365+
let probe_scan = TestScanBuilder::new(Arc::clone(&probe_schema))
3366+
.with_support(true)
3367+
.with_batches(probe_batches)
3368+
.build();
3369+
3370+
// ProjectionExec reorders (a, value) → (value, a)
3371+
let reordered_schema = Arc::new(Schema::new(vec![
3372+
Field::new("value", DataType::Float64, false),
3373+
Field::new("a", DataType::Utf8, false),
3374+
]));
3375+
let projection = Arc::new(
3376+
ProjectionExec::try_new(
3377+
vec![
3378+
(col("value", &probe_schema).unwrap(), "value".to_string()),
3379+
(col("a", &probe_schema).unwrap(), "a".to_string()),
3380+
],
3381+
probe_scan,
3382+
)
3383+
.unwrap(),
3384+
);
3385+
3386+
// AggregateExec: GROUP BY a@1, min(value@0)
3387+
let aggregate_expr = vec![
3388+
AggregateExprBuilder::new(
3389+
min_udaf(),
3390+
vec![col("value", &reordered_schema).unwrap()],
3391+
)
3392+
.schema(reordered_schema.clone())
3393+
.alias("min_value")
3394+
.build()
3395+
.map(Arc::new)
3396+
.unwrap(),
3397+
];
3398+
let group_by = PhysicalGroupBy::new_single(vec![(
3399+
col("a", &reordered_schema).unwrap(), // a@1 in input
3400+
"a".to_string(),
3401+
)]);
3402+
3403+
let aggregate = Arc::new(
3404+
AggregateExec::try_new(
3405+
AggregateMode::Single,
3406+
group_by,
3407+
aggregate_expr,
3408+
vec![None],
3409+
projection,
3410+
reordered_schema,
3411+
)
3412+
.unwrap(),
3413+
);
3414+
3415+
// Aggregate output schema: (a@0, min_value@1)
3416+
let agg_output_schema = aggregate.schema();
3417+
3418+
// Join the build and probe side
3419+
let plan = Arc::new(
3420+
HashJoinExec::try_new(
3421+
build_scan,
3422+
aggregate,
3423+
vec![(
3424+
col("a", &build_schema).unwrap(),
3425+
col("a", &agg_output_schema).unwrap(),
3426+
)],
3427+
None,
3428+
&JoinType::Inner,
3429+
None,
3430+
PartitionMode::CollectLeft,
3431+
datafusion_common::NullEquality::NullEqualsNothing,
3432+
false,
3433+
)
3434+
.unwrap(),
3435+
) as Arc<dyn ExecutionPlan>;
3436+
3437+
// The HashJoin's dynamic filter on `a` should push
3438+
// through the aggregate and reach the probe-side DataSource.
3439+
insta::assert_snapshot!(
3440+
OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new_post_optimization(), true),
3441+
@r"
3442+
OptimizationTest:
3443+
input:
3444+
- HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0)]
3445+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a], file_type=test, pushdown_supported=true
3446+
- AggregateExec: mode=Single, gby=[a@1 as a], aggr=[min_value]
3447+
- ProjectionExec: expr=[value@1 as value, a@0 as a]
3448+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, value], file_type=test, pushdown_supported=true
3449+
output:
3450+
Ok:
3451+
- HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0)]
3452+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a], file_type=test, pushdown_supported=true
3453+
- AggregateExec: mode=Single, gby=[a@1 as a], aggr=[min_value]
3454+
- ProjectionExec: expr=[value@1 as value, a@0 as a]
3455+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, value], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]
3456+
"
3457+
);
3458+
3459+
// Actually execute the plan to verify the dynamic filter is populated
3460+
let mut config = ConfigOptions::default();
3461+
config.execution.parquet.pushdown_filters = true;
3462+
let plan = FilterPushdown::new_post_optimization()
3463+
.optimize(plan, &config)
3464+
.unwrap();
3465+
3466+
let session_config = SessionConfig::new().with_batch_size(10);
3467+
let session_ctx = SessionContext::new_with_config(session_config);
3468+
session_ctx.register_object_store(
3469+
ObjectStoreUrl::parse("test://").unwrap().as_ref(),
3470+
Arc::new(InMemory::new()),
3471+
);
3472+
let state = session_ctx.state();
3473+
let task_ctx = state.task_ctx();
3474+
let mut stream = plan.execute(0, Arc::clone(&task_ctx)).unwrap();
3475+
stream.next().await.unwrap().unwrap();
3476+
3477+
// After execution, the dynamic filter should be populated with values
3478+
insta::assert_snapshot!(
3479+
format!("{}", format_plan_for_test(&plan)),
3480+
@r"
3481+
- HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0)]
3482+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a], file_type=test, pushdown_supported=true
3483+
- AggregateExec: mode=Single, gby=[a@1 as a], aggr=[min_value]
3484+
- ProjectionExec: expr=[value@1 as value, a@0 as a]
3485+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, value], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 >= h1 AND a@0 <= h2 AND a@0 IN (SET) ([h1, h2]) ]
3486+
"
3487+
);
3488+
}
3489+
31483490
#[test]
31493491
fn test_pushdown_with_computed_grouping_key() {
31503492
// Test filter pushdown with computed grouping expression

0 commit comments

Comments
 (0)