Skip to content

Commit 106653a

Browse files
alambclaude
andcommitted
Gate static conjuncts only when a dynamic filter is present
Refines the narrow-projection pushdown gate: previously any scan with a dynamic filter (e.g. TopK's threshold on `ORDER BY .. LIMIT` queries) bypassed the gate entirely, so static conjuncts were still pushed as a RowFilter on narrow projections -- reintroducing the regression the gate exists to prevent (ClickBench Q24/Q25/Q26: `SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY .. LIMIT 10`). Now, for a narrow projection with a dynamic filter: - pushdown stays enabled and the dynamic conjuncts are still pushed (the TopK threshold needs the scan-time RowFilter / RowGroupPruner cascade), - the static conjuncts are declined: parents keep them in a FilterExec, and the opener builds the RowFilter from only the dynamic-filter conjuncts of the scan predicate (no double evaluation), - static conjuncts remain in the scan predicate for stats / bloom / page-index pruning. Implemented via a new internal `pushdown_dynamic_filters_only` marker on `ParquetSource`, threaded through the morselizer to the opener, which splits the predicate conjuncts by `DynamicFilterTracking` when building the row filter. Co-Authored-By: Claude Fable 5 <[email protected]>
1 parent 026fb7d commit 106653a

2 files changed

Lines changed: 199 additions & 15 deletions

File tree

‎datafusion/datasource-parquet/src/opener/mod.rs‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ use datafusion_common::{
6060
use datafusion_datasource::{PartitionedFile, TableSchema};
6161
use datafusion_physical_expr::expressions::{Column, DynamicFilterTracking};
6262
use datafusion_physical_expr::simplifier::PhysicalExprSimplifier;
63+
use datafusion_physical_expr::utils::{conjunction_opt, split_conjunction};
6364
use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;
6465
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
6566
use datafusion_physical_expr_common::sort_expr::LexOrdering;
@@ -260,6 +261,11 @@ pub(super) struct ParquetMorselizer {
260261
/// Should the filters be evaluated during the parquet scan using
261262
/// [`DatafusionArrowPredicate`](crate::row_filter::DatafusionArrowPredicate)?
262263
pub pushdown_filters: bool,
264+
/// When true (and `pushdown_filters` is enabled), build the scan's
265+
/// RowFilter from only the dynamic-filter conjuncts of the predicate;
266+
/// static conjuncts stay in the `FilterExec` above the scan. Set by
267+
/// the narrow-projection gate in `ParquetSource::try_pushdown_filters`.
268+
pub(crate) pushdown_dynamic_filters_only: bool,
263269
/// Should the filters be reordered to optimize the scan?
264270
pub reorder_filters: bool,
265271
/// Should we force the reader to use RowSelections for filtering
@@ -452,6 +458,7 @@ struct PreparedParquetOpen {
452458
virtual_state: Option<Arc<VirtualColumnsState>>,
453459
reorder_predicates: bool,
454460
pushdown_filters: bool,
461+
pushdown_dynamic_filters_only: bool,
455462
force_filter_selections: bool,
456463
progressive_io: bool,
457464
enable_page_index: bool,
@@ -850,6 +857,7 @@ impl ParquetMorselizer {
850857
virtual_state: self.virtual_state.as_ref().map(Arc::clone),
851858
reorder_predicates: self.reorder_filters,
852859
pushdown_filters: self.pushdown_filters,
860+
pushdown_dynamic_filters_only: self.pushdown_dynamic_filters_only,
853861
force_filter_selections: self.force_filter_selections,
854862
progressive_io: self.progressive_io,
855863
enable_page_index: self.enable_page_index,
@@ -1444,10 +1452,37 @@ impl RowGroupsPrunedParquetOpen {
14441452
)?;
14451453

14461454
let (decoder, rg_plan, one_shot_ranges) = {
1447-
let pushdown_predicate = prepared
1448-
.pushdown_filters
1455+
// When the narrow-projection gate declined the static conjuncts
1456+
// but kept pushdown enabled for a dynamic filter
1457+
// (`pushdown_dynamic_filters_only`), build the RowFilter from
1458+
// only the dynamic-filter conjuncts: the static conjuncts are
1459+
// evaluated by the `FilterExec` above the scan, so also running
1460+
// them here would evaluate them twice per batch. They remain in
1461+
// `prepared.predicate`, where they drive stats / bloom /
1462+
// page-index pruning.
1463+
let dynamic_only_predicate = (prepared.pushdown_filters
1464+
&& prepared.pushdown_dynamic_filters_only)
14491465
.then_some(prepared.predicate.as_ref())
1450-
.flatten();
1466+
.flatten()
1467+
.and_then(|predicate| {
1468+
conjunction_opt(
1469+
split_conjunction(predicate)
1470+
.into_iter()
1471+
.map(Arc::clone)
1472+
.filter(|conjunct| {
1473+
DynamicFilterTracking::classify(conjunct)
1474+
.contains_dynamic_filter()
1475+
}),
1476+
)
1477+
});
1478+
let pushdown_predicate = if prepared.pushdown_dynamic_filters_only {
1479+
dynamic_only_predicate.as_ref()
1480+
} else {
1481+
prepared
1482+
.pushdown_filters
1483+
.then_some(prepared.predicate.as_ref())
1484+
.flatten()
1485+
};
14511486
let mut row_filter_generator = RowFilterGenerator::new(
14521487
pushdown_predicate,
14531488
&prepared.physical_file_schema,
@@ -1832,6 +1867,7 @@ mod test {
18321867
metrics: ExecutionPlanMetricsSet,
18331868
parquet_file_reader_factory: Option<Arc<dyn ParquetFileReaderFactory>>,
18341869
pushdown_filters: bool,
1870+
pushdown_dynamic_filters_only: bool,
18351871
reorder_filters: bool,
18361872
force_filter_selections: bool,
18371873
progressive_io: bool,
@@ -2044,6 +2080,7 @@ mod test {
20442080
metrics: ExecutionPlanMetricsSet::new(),
20452081
parquet_file_reader_factory: None,
20462082
pushdown_filters: false,
2083+
pushdown_dynamic_filters_only: false,
20472084
reorder_filters: false,
20482085
force_filter_selections: false,
20492086
progressive_io: false,
@@ -2214,6 +2251,7 @@ mod test {
22142251
Arc::new(DefaultParquetFileReaderFactory::new(store)) as _
22152252
}),
22162253
pushdown_filters: self.pushdown_filters,
2254+
pushdown_dynamic_filters_only: self.pushdown_dynamic_filters_only,
22172255
reorder_filters: self.reorder_filters,
22182256
force_filter_selections: self.force_filter_selections,
22192257
progressive_io: self.progressive_io,

‎datafusion/datasource-parquet/src/source.rs‎

Lines changed: 158 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,15 @@ pub struct ParquetSource {
309309
pub(crate) projection: ProjectionExprs,
310310
#[cfg(feature = "parquet_encryption")]
311311
pub(crate) encryption_factory: Option<Arc<dyn EncryptionFactory>>,
312+
/// When true (set by the narrow-projection gate in
313+
/// [`Self::try_pushdown_filters`]), the scan installs a `RowFilter`
314+
/// built from only the dynamic-filter conjuncts of `predicate`. The
315+
/// static conjuncts were declined by the gate and stay in the
316+
/// `FilterExec` above the scan, so also running them in the scan's
317+
/// `RowFilter` would evaluate them twice; they remain part of
318+
/// `predicate` for stats / bloom / page-index pruning. Only
319+
/// meaningful when `pushdown_filters` is enabled.
320+
pub(crate) pushdown_dynamic_filters_only: bool,
312321
/// If true, the opener flips row-group iteration order. Within-
313322
/// row-group order is on-disk order, so the scan is `Inexact` and
314323
/// a `SortExec` is kept in the plan.
@@ -340,6 +349,7 @@ impl ParquetSource {
340349
metadata_size_hint: None,
341350
#[cfg(feature = "parquet_encryption")]
342351
encryption_factory: None,
352+
pushdown_dynamic_filters_only: false,
343353
reverse_row_groups: false,
344354
sort_order_for_reorder: None,
345355
}
@@ -660,6 +670,7 @@ impl FileSource for ParquetSource {
660670
metrics: self.metrics().clone(),
661671
parquet_file_reader_factory,
662672
pushdown_filters: self.pushdown_filters(),
673+
pushdown_dynamic_filters_only: self.pushdown_dynamic_filters_only,
663674
reorder_filters: self.reorder_filters(),
664675
force_filter_selections: self.force_filter_selections(),
665676
progressive_io: self.progressive_io(),
@@ -881,16 +892,20 @@ impl FileSource for ParquetSource {
881892
//
882893
// [#22883]: https://github.com/apache/datafusion/issues/22883
883894
const PUSHDOWN_MIN_NON_FILTER_COLS: usize = 3;
884-
// Never gate a scan whose predicate already contains a dynamic
885-
// filter — either in the incoming `filters` parameter, or already
886-
// installed on `self.predicate` by an earlier optimizer rule (this
887-
// is how `TopK`'s heap-threshold expression arrives: the sort
888-
// pushdown / TopK rule injects the `DynamicFilterPhysicalExpr`
889-
// into the scan's predicate before filter pushdown runs). Dynamic
890-
// filters rely on the scan-time RowFilter/RowGroupPruner cascade
891-
// to prune data as the threshold tightens, so declining pushdown
892-
// here would silently disable the entire dynamic-RG-prune path
893-
// for narrow `ORDER BY ... LIMIT` queries.
895+
// When the scan involves a dynamic filter — either in the incoming
896+
// `filters` parameter, or already installed on `self.predicate` by
897+
// an earlier optimizer rule (this is how `TopK`'s heap-threshold
898+
// expression arrives: the sort pushdown / TopK rule injects the
899+
// `DynamicFilterPhysicalExpr` into the scan's predicate before
900+
// filter pushdown runs) — the gate must not disable pushdown
901+
// wholesale: dynamic filters rely on the scan-time
902+
// RowFilter/RowGroupPruner cascade to prune data as the threshold
903+
// tightens. Instead, for a narrow projection the gate declines
904+
// only the *static* conjuncts (they stay in the `FilterExec`
905+
// above) and keeps pushdown enabled for the dynamic conjuncts by
906+
// setting `pushdown_dynamic_filters_only`; the opener then builds
907+
// the RowFilter from only the dynamic-filter conjuncts of the
908+
// scan predicate.
894909
let existing_predicate_has_dynamic = self.predicate.as_ref().is_some_and(|p| {
895910
DynamicFilterTracking::classify(p).contains_dynamic_filter()
896911
});
@@ -899,8 +914,8 @@ impl FileSource for ParquetSource {
899914
.any(|f| DynamicFilterTracking::classify(f).contains_dynamic_filter());
900915
let has_dynamic_filter =
901916
existing_predicate_has_dynamic || incoming_filters_have_dynamic;
917+
let mut pushdown_dynamic_filters_only = false;
902918
if pushdown_filters
903-
&& !has_dynamic_filter
904919
&& config.execution.parquet.pushdown_filter_mode
905920
!= ParquetPushdownFilterMode::Always
906921
{
@@ -924,7 +939,11 @@ impl FileSource for ParquetSource {
924939
.collect::<std::collections::HashSet<_>>()
925940
.len();
926941
if non_filter_projected < PUSHDOWN_MIN_NON_FILTER_COLS {
927-
pushdown_filters = false;
942+
if has_dynamic_filter {
943+
pushdown_dynamic_filters_only = true;
944+
} else {
945+
pushdown_filters = false;
946+
}
928947
}
929948
}
930949

@@ -984,6 +1003,9 @@ impl FileSource for ParquetSource {
9841003
// re-enabled). So flipping the flag to `false` here does not
9851004
// permanently disable pushdown for future calls.
9861005
source = source.with_pushdown_filters(pushdown_filters);
1006+
// Recomputed from scratch on every call, so a decision made for an
1007+
// earlier filter shape never leaks into later calls.
1008+
source.pushdown_dynamic_filters_only = pushdown_dynamic_filters_only;
9871009
// `progressive_io` resolves table-or-session, the same way as
9881010
// `pushdown_filters` above
9891011
source = source.with_progressive_io(
@@ -998,6 +1020,30 @@ impl FileSource for ParquetSource {
9981020
)
9991021
.with_updated_node(source));
10001022
}
1023+
if pushdown_dynamic_filters_only {
1024+
// The gate declined the static conjuncts: parents keep them (a
1025+
// `FilterExec` above the scan evaluates them), and only
1026+
// dynamic-filter conjuncts are pushed into the scan's
1027+
// RowFilter. The static conjuncts were still merged into
1028+
// `source.predicate` above, where they drive stats / bloom /
1029+
// page-index pruning but not row filtering.
1030+
return Ok(FilterPushdownPropagation::with_parent_pushdown_result(
1031+
filters
1032+
.iter()
1033+
.map(|f| {
1034+
if matches!(f.discriminant, PushedDown::Yes)
1035+
&& DynamicFilterTracking::classify(&f.predicate)
1036+
.contains_dynamic_filter()
1037+
{
1038+
PushedDown::Yes
1039+
} else {
1040+
PushedDown::No
1041+
}
1042+
})
1043+
.collect(),
1044+
)
1045+
.with_updated_node(source));
1046+
}
10011047
Ok(FilterPushdownPropagation::with_parent_pushdown_result(
10021048
filters.iter().map(|f| f.discriminant).collect(),
10031049
)
@@ -2431,5 +2477,105 @@ mod tests {
24312477
the `||` recovery against the config default is what makes this \
24322478
safe after the first call flipped the flag to false"
24332479
);
2480+
assert!(
2481+
after_dynamic.pushdown_dynamic_filters_only,
2482+
"narrow projection with a dynamic filter must push only the \
2483+
dynamic conjuncts, not disable the gate entirely"
2484+
);
2485+
}
2486+
2487+
/// When the projection is narrow AND the scan involves a dynamic filter
2488+
/// (the TopK shape, e.g. ClickBench Q24-Q26:
2489+
/// `SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> ''
2490+
/// ORDER BY "EventTime" LIMIT 10`), the gate must:
2491+
///
2492+
/// 1. keep `pushdown_filters` enabled (the dynamic threshold needs the
2493+
/// scan-time RowFilter / RowGroupPruner cascade),
2494+
/// 2. mark the source `pushdown_dynamic_filters_only` so the opener
2495+
/// builds the RowFilter from only the dynamic conjuncts, and
2496+
/// 3. report `PushedDown::No` for the *static* filters so the
2497+
/// `FilterExec` above the scan keeps evaluating them (no double
2498+
/// filtering: they are excluded from the scan's RowFilter).
2499+
///
2500+
/// Before this change the gate skipped narrow-projection scans with a
2501+
/// dynamic filter entirely, so the static conjuncts were still pushed
2502+
/// as a RowFilter — reintroducing exactly the narrow-projection
2503+
/// regression the gate exists to prevent.
2504+
#[test]
2505+
fn test_narrow_projection_gate_with_dynamic_filter_pushes_only_dynamic() {
2506+
use arrow::datatypes::{DataType, Field, Schema};
2507+
use datafusion_common::config::ConfigOptions;
2508+
use datafusion_datasource::TableSchema;
2509+
use datafusion_expr::{col, lit as logical_lit};
2510+
use datafusion_physical_expr::expressions::{DynamicFilterPhysicalExpr, lit};
2511+
use datafusion_physical_expr::planner::logical2physical;
2512+
use datafusion_physical_plan::filter_pushdown::PushedDown;
2513+
2514+
let file_schema = Arc::new(Schema::new(vec![
2515+
Field::new("a", DataType::Int64, false),
2516+
Field::new("b", DataType::Int64, false),
2517+
]));
2518+
let table_schema = TableSchema::from(file_schema);
2519+
let full_schema: arrow::datatypes::SchemaRef =
2520+
Arc::clone(table_schema.table_schema());
2521+
2522+
// Narrow projection: columns `a`, `b`; static filter on `a` leaves
2523+
// one non-filter projected column (< PUSHDOWN_MIN_NON_FILTER_COLS).
2524+
let projection = ProjectionExprs::from_indices(&[0usize, 1], &full_schema);
2525+
let mut source = ParquetSource::new(table_schema).with_pushdown_filters(true);
2526+
source.projection = projection;
2527+
2528+
// Simulate the TopK rule having already installed its dynamic
2529+
// threshold filter on the scan's predicate.
2530+
let dyn_col = Arc::new(Column::new("b", 1)) as Arc<dyn PhysicalExpr>;
2531+
source.predicate = Some(Arc::new(DynamicFilterPhysicalExpr::new(
2532+
vec![dyn_col],
2533+
lit(true) as Arc<dyn PhysicalExpr>,
2534+
)) as Arc<dyn PhysicalExpr>);
2535+
2536+
let static_filter =
2537+
logical2physical(&col("a").eq(logical_lit(1i64)), &full_schema);
2538+
let config = ConfigOptions::default();
2539+
assert!(config.execution.parquet.pushdown_filters);
2540+
2541+
let prop = source
2542+
.try_pushdown_filters(vec![static_filter], &config)
2543+
.expect("try_pushdown_filters must not error");
2544+
2545+
// The static filter must stay with the parent (FilterExec).
2546+
assert_eq!(prop.filters.len(), 1);
2547+
assert!(
2548+
matches!(prop.filters[0], PushedDown::No),
2549+
"static conjunct must be declined so the FilterExec above the \
2550+
scan keeps it"
2551+
);
2552+
2553+
let updated = prop
2554+
.updated_node
2555+
.as_ref()
2556+
.expect("dynamic-only gate path must attach an updated source")
2557+
.downcast_ref::<ParquetSource>()
2558+
.expect("updated node must be a ParquetSource");
2559+
assert!(
2560+
updated.pushdown_filters(),
2561+
"pushdown must stay enabled so the dynamic threshold reaches \
2562+
the scan-time RowFilter / RowGroupPruner cascade"
2563+
);
2564+
assert!(
2565+
updated.pushdown_dynamic_filters_only,
2566+
"the source must be marked dynamic-only so the opener excludes \
2567+
the static conjuncts from the RowFilter"
2568+
);
2569+
// The static conjunct must still be merged into the scan predicate
2570+
// (for stats / bloom / page-index pruning).
2571+
let predicate = updated
2572+
.predicate
2573+
.as_ref()
2574+
.expect("predicate must be present");
2575+
assert!(
2576+
format!("{predicate:?}").contains("BinaryExpr"),
2577+
"static conjunct should be merged into the scan predicate for \
2578+
pruning, got: {predicate:?}"
2579+
);
24342580
}
24352581
}

0 commit comments

Comments
 (0)