@@ -309,6 +309,15 @@ pub struct ParquetSource {
309309 pub ( crate ) projection : ProjectionExprs ,
310310 #[ cfg( feature = "parquet_encryption" ) ]
311311 pub ( crate ) encryption_factory : Option < Arc < dyn EncryptionFactory > > ,
312+ /// When true (set by the narrow-projection gate in
313+ /// [`Self::try_pushdown_filters`]), the scan installs a `RowFilter`
314+ /// built from only the dynamic-filter conjuncts of `predicate`. The
315+ /// static conjuncts were declined by the gate and stay in the
316+ /// `FilterExec` above the scan, so also running them in the scan's
317+ /// `RowFilter` would evaluate them twice; they remain part of
318+ /// `predicate` for stats / bloom / page-index pruning. Only
319+ /// meaningful when `pushdown_filters` is enabled.
320+ pub ( crate ) pushdown_dynamic_filters_only : bool ,
312321 /// If true, the opener flips row-group iteration order. Within-
313322 /// row-group order is on-disk order, so the scan is `Inexact` and
314323 /// a `SortExec` is kept in the plan.
@@ -340,6 +349,7 @@ impl ParquetSource {
340349 metadata_size_hint : None ,
341350 #[ cfg( feature = "parquet_encryption" ) ]
342351 encryption_factory : None ,
352+ pushdown_dynamic_filters_only : false ,
343353 reverse_row_groups : false ,
344354 sort_order_for_reorder : None ,
345355 }
@@ -660,6 +670,7 @@ impl FileSource for ParquetSource {
660670 metrics : self . metrics ( ) . clone ( ) ,
661671 parquet_file_reader_factory,
662672 pushdown_filters : self . pushdown_filters ( ) ,
673+ pushdown_dynamic_filters_only : self . pushdown_dynamic_filters_only ,
663674 reorder_filters : self . reorder_filters ( ) ,
664675 force_filter_selections : self . force_filter_selections ( ) ,
665676 progressive_io : self . progressive_io ( ) ,
@@ -881,16 +892,20 @@ impl FileSource for ParquetSource {
881892 //
882893 // [#22883]: https://github.com/apache/datafusion/issues/22883
883894 const PUSHDOWN_MIN_NON_FILTER_COLS : usize = 3 ;
884- // Never gate a scan whose predicate already contains a dynamic
885- // filter — either in the incoming `filters` parameter, or already
886- // installed on `self.predicate` by an earlier optimizer rule (this
887- // is how `TopK`'s heap-threshold expression arrives: the sort
888- // pushdown / TopK rule injects the `DynamicFilterPhysicalExpr`
889- // into the scan's predicate before filter pushdown runs). Dynamic
890- // filters rely on the scan-time RowFilter/RowGroupPruner cascade
891- // to prune data as the threshold tightens, so declining pushdown
892- // here would silently disable the entire dynamic-RG-prune path
893- // for narrow `ORDER BY ... LIMIT` queries.
895+ // When the scan involves a dynamic filter — either in the incoming
896+ // `filters` parameter, or already installed on `self.predicate` by
897+ // an earlier optimizer rule (this is how `TopK`'s heap-threshold
898+ // expression arrives: the sort pushdown / TopK rule injects the
899+ // `DynamicFilterPhysicalExpr` into the scan's predicate before
900+ // filter pushdown runs) — the gate must not disable pushdown
901+ // wholesale: dynamic filters rely on the scan-time
902+ // RowFilter/RowGroupPruner cascade to prune data as the threshold
903+ // tightens. Instead, for a narrow projection the gate declines
904+ // only the *static* conjuncts (they stay in the `FilterExec`
905+ // above) and keeps pushdown enabled for the dynamic conjuncts by
906+ // setting `pushdown_dynamic_filters_only`; the opener then builds
907+ // the RowFilter from only the dynamic-filter conjuncts of the
908+ // scan predicate.
894909 let existing_predicate_has_dynamic = self . predicate . as_ref ( ) . is_some_and ( |p| {
895910 DynamicFilterTracking :: classify ( p) . contains_dynamic_filter ( )
896911 } ) ;
@@ -899,8 +914,8 @@ impl FileSource for ParquetSource {
899914 . any ( |f| DynamicFilterTracking :: classify ( f) . contains_dynamic_filter ( ) ) ;
900915 let has_dynamic_filter =
901916 existing_predicate_has_dynamic || incoming_filters_have_dynamic;
917+ let mut pushdown_dynamic_filters_only = false ;
902918 if pushdown_filters
903- && !has_dynamic_filter
904919 && config. execution . parquet . pushdown_filter_mode
905920 != ParquetPushdownFilterMode :: Always
906921 {
@@ -924,7 +939,11 @@ impl FileSource for ParquetSource {
924939 . collect :: < std:: collections:: HashSet < _ > > ( )
925940 . len ( ) ;
926941 if non_filter_projected < PUSHDOWN_MIN_NON_FILTER_COLS {
927- pushdown_filters = false ;
942+ if has_dynamic_filter {
943+ pushdown_dynamic_filters_only = true ;
944+ } else {
945+ pushdown_filters = false ;
946+ }
928947 }
929948 }
930949
@@ -984,6 +1003,9 @@ impl FileSource for ParquetSource {
9841003 // re-enabled). So flipping the flag to `false` here does not
9851004 // permanently disable pushdown for future calls.
9861005 source = source. with_pushdown_filters ( pushdown_filters) ;
1006+ // Recomputed from scratch on every call, so a decision made for an
1007+ // earlier filter shape never leaks into later calls.
1008+ source. pushdown_dynamic_filters_only = pushdown_dynamic_filters_only;
9871009 // `progressive_io` resolves table-or-session, the same way as
9881010 // `pushdown_filters` above
9891011 source = source. with_progressive_io (
@@ -998,6 +1020,30 @@ impl FileSource for ParquetSource {
9981020 )
9991021 . with_updated_node ( source) ) ;
10001022 }
1023+ if pushdown_dynamic_filters_only {
1024+ // The gate declined the static conjuncts: parents keep them (a
1025+ // `FilterExec` above the scan evaluates them), and only
1026+ // dynamic-filter conjuncts are pushed into the scan's
1027+ // RowFilter. The static conjuncts were still merged into
1028+ // `source.predicate` above, where they drive stats / bloom /
1029+ // page-index pruning but not row filtering.
1030+ return Ok ( FilterPushdownPropagation :: with_parent_pushdown_result (
1031+ filters
1032+ . iter ( )
1033+ . map ( |f| {
1034+ if matches ! ( f. discriminant, PushedDown :: Yes )
1035+ && DynamicFilterTracking :: classify ( & f. predicate )
1036+ . contains_dynamic_filter ( )
1037+ {
1038+ PushedDown :: Yes
1039+ } else {
1040+ PushedDown :: No
1041+ }
1042+ } )
1043+ . collect ( ) ,
1044+ )
1045+ . with_updated_node ( source) ) ;
1046+ }
10011047 Ok ( FilterPushdownPropagation :: with_parent_pushdown_result (
10021048 filters. iter ( ) . map ( |f| f. discriminant ) . collect ( ) ,
10031049 )
@@ -2431,5 +2477,105 @@ mod tests {
24312477 the `||` recovery against the config default is what makes this \
24322478 safe after the first call flipped the flag to false"
24332479 ) ;
2480+ assert ! (
2481+ after_dynamic. pushdown_dynamic_filters_only,
2482+ "narrow projection with a dynamic filter must push only the \
2483+ dynamic conjuncts, not disable the gate entirely"
2484+ ) ;
2485+ }
2486+
2487+ /// When the projection is narrow AND the scan involves a dynamic filter
2488+ /// (the TopK shape, e.g. ClickBench Q24-Q26:
2489+ /// `SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> ''
2490+ /// ORDER BY "EventTime" LIMIT 10`), the gate must:
2491+ ///
2492+ /// 1. keep `pushdown_filters` enabled (the dynamic threshold needs the
2493+ /// scan-time RowFilter / RowGroupPruner cascade),
2494+ /// 2. mark the source `pushdown_dynamic_filters_only` so the opener
2495+ /// builds the RowFilter from only the dynamic conjuncts, and
2496+ /// 3. report `PushedDown::No` for the *static* filters so the
2497+ /// `FilterExec` above the scan keeps evaluating them (no double
2498+ /// filtering: they are excluded from the scan's RowFilter).
2499+ ///
2500+ /// Before this change the gate skipped narrow-projection scans with a
2501+ /// dynamic filter entirely, so the static conjuncts were still pushed
2502+ /// as a RowFilter — reintroducing exactly the narrow-projection
2503+ /// regression the gate exists to prevent.
2504+ #[ test]
2505+ fn test_narrow_projection_gate_with_dynamic_filter_pushes_only_dynamic ( ) {
2506+ use arrow:: datatypes:: { DataType , Field , Schema } ;
2507+ use datafusion_common:: config:: ConfigOptions ;
2508+ use datafusion_datasource:: TableSchema ;
2509+ use datafusion_expr:: { col, lit as logical_lit} ;
2510+ use datafusion_physical_expr:: expressions:: { DynamicFilterPhysicalExpr , lit} ;
2511+ use datafusion_physical_expr:: planner:: logical2physical;
2512+ use datafusion_physical_plan:: filter_pushdown:: PushedDown ;
2513+
2514+ let file_schema = Arc :: new ( Schema :: new ( vec ! [
2515+ Field :: new( "a" , DataType :: Int64 , false ) ,
2516+ Field :: new( "b" , DataType :: Int64 , false ) ,
2517+ ] ) ) ;
2518+ let table_schema = TableSchema :: from ( file_schema) ;
2519+ let full_schema: arrow:: datatypes:: SchemaRef =
2520+ Arc :: clone ( table_schema. table_schema ( ) ) ;
2521+
2522+ // Narrow projection: columns `a`, `b`; static filter on `a` leaves
2523+ // one non-filter projected column (< PUSHDOWN_MIN_NON_FILTER_COLS).
2524+ let projection = ProjectionExprs :: from_indices ( & [ 0usize , 1 ] , & full_schema) ;
2525+ let mut source = ParquetSource :: new ( table_schema) . with_pushdown_filters ( true ) ;
2526+ source. projection = projection;
2527+
2528+ // Simulate the TopK rule having already installed its dynamic
2529+ // threshold filter on the scan's predicate.
2530+ let dyn_col = Arc :: new ( Column :: new ( "b" , 1 ) ) as Arc < dyn PhysicalExpr > ;
2531+ source. predicate = Some ( Arc :: new ( DynamicFilterPhysicalExpr :: new (
2532+ vec ! [ dyn_col] ,
2533+ lit ( true ) as Arc < dyn PhysicalExpr > ,
2534+ ) ) as Arc < dyn PhysicalExpr > ) ;
2535+
2536+ let static_filter =
2537+ logical2physical ( & col ( "a" ) . eq ( logical_lit ( 1i64 ) ) , & full_schema) ;
2538+ let config = ConfigOptions :: default ( ) ;
2539+ assert ! ( config. execution. parquet. pushdown_filters) ;
2540+
2541+ let prop = source
2542+ . try_pushdown_filters ( vec ! [ static_filter] , & config)
2543+ . expect ( "try_pushdown_filters must not error" ) ;
2544+
2545+ // The static filter must stay with the parent (FilterExec).
2546+ assert_eq ! ( prop. filters. len( ) , 1 ) ;
2547+ assert ! (
2548+ matches!( prop. filters[ 0 ] , PushedDown :: No ) ,
2549+ "static conjunct must be declined so the FilterExec above the \
2550+ scan keeps it"
2551+ ) ;
2552+
2553+ let updated = prop
2554+ . updated_node
2555+ . as_ref ( )
2556+ . expect ( "dynamic-only gate path must attach an updated source" )
2557+ . downcast_ref :: < ParquetSource > ( )
2558+ . expect ( "updated node must be a ParquetSource" ) ;
2559+ assert ! (
2560+ updated. pushdown_filters( ) ,
2561+ "pushdown must stay enabled so the dynamic threshold reaches \
2562+ the scan-time RowFilter / RowGroupPruner cascade"
2563+ ) ;
2564+ assert ! (
2565+ updated. pushdown_dynamic_filters_only,
2566+ "the source must be marked dynamic-only so the opener excludes \
2567+ the static conjuncts from the RowFilter"
2568+ ) ;
2569+ // The static conjunct must still be merged into the scan predicate
2570+ // (for stats / bloom / page-index pruning).
2571+ let predicate = updated
2572+ . predicate
2573+ . as_ref ( )
2574+ . expect ( "predicate must be present" ) ;
2575+ assert ! (
2576+ format!( "{predicate:?}" ) . contains( "BinaryExpr" ) ,
2577+ "static conjunct should be merged into the scan predicate for \
2578+ pruning, got: {predicate:?}"
2579+ ) ;
24342580 }
24352581}
0 commit comments