@@ -854,7 +854,6 @@ impl FileSource for ParquetSource {
854854 //
855855 // [#22883]: https://github.com/apache/datafusion/issues/22883
856856 const PUSHDOWN_MIN_NON_FILTER_COLS : usize = 3 ;
857- let mut narrow_projection_gate_declined = false ;
858857 // Never gate a scan whose predicate already contains a dynamic
859858 // filter — either in the incoming `filters` parameter, or already
860859 // installed on `self.predicate` by an earlier optimizer rule (this
@@ -899,7 +898,6 @@ impl FileSource for ParquetSource {
899898 . len ( ) ;
900899 if non_filter_projected < PUSHDOWN_MIN_NON_FILTER_COLS {
901900 pushdown_filters = false ;
902- narrow_projection_gate_declined = true ;
903901 }
904902 }
905903
@@ -938,18 +936,27 @@ impl FileSource for ParquetSource {
938936 None => conjunction ( allowed_filters) ,
939937 } ;
940938 source. predicate = Some ( predicate) ;
941- // Only persist the pushdown_filters bit on the source when this is
942- // a "real" config-driven decision. When the gate declined, we
943- // deliberately do NOT flip the source's pushdown flag to false —
944- // a later `try_pushdown_filters` call (e.g. from TopK's dynamic
945- // filter injection) needs the source to still be pushdown-eligible
946- // so that dynamic filter can install its RowFilter and drive the
947- // runtime RG-prune cascade. The current call still returns
948- // `PushedDown::No` for its incoming filters, so the FilterExec
949- // above the scan stays and correctness is preserved.
950- if !narrow_projection_gate_declined {
951- source = source. with_pushdown_filters ( pushdown_filters) ;
952- }
939+ // Always persist the effective pushdown decision on the source.
940+ // When the gate declined, `pushdown_filters` is already `false`
941+ // here (set above by the gate), so `source.pushdown_filters`
942+ // becomes `false` and the scan will NOT install a RowFilter —
943+ // avoiding a "double filter" (FilterExec above the scan + RowFilter
944+ // inside the scan running the same conjuncts on every batch).
945+ //
946+ // A later `try_pushdown_filters` call (e.g. from TopK's dynamic
947+ // filter injection) re-computes `pushdown_filters` from scratch:
948+ //
949+ // let pushdown_filters =
950+ // table_pushdown_enabled || config_pushdown_enabled;
951+ //
952+ // Because the config default is `true`, `||` restores `true` even
953+ // if a previous call flipped the source-level flag to `false`.
954+ // The gate in the next call then re-decides based on the new
955+ // filter shape (`has_dynamic_filter` is true for TopK's injected
956+ // filter, so the gate skips the decline path and pushdown is
957+ // re-enabled). So flipping the flag to `false` here does not
958+ // permanently disable pushdown for future calls.
959+ source = source. with_pushdown_filters ( pushdown_filters) ;
953960 let source = Arc :: new ( source) ;
954961 // If pushdown_filters is false we tell our parents that they still have to handle the filters,
955962 // even if we updated the predicate to include the filters (they will only be used for stats pruning).
@@ -1983,4 +1990,208 @@ mod tests {
19831990 pushed down"
19841991 ) ;
19851992 }
1993+
1994+ /// Regression test for the "double filter" bug on the narrow-projection
1995+ /// gate.
1996+ ///
1997+ /// Before the fix: when the gate decided to decline pushdown for a
1998+ /// narrow projection, the code deliberately skipped calling
1999+ /// `source.with_pushdown_filters(false)`. That left the source-level
2000+ /// `pushdown_filters` flag at whatever it was (which is `true` by
2001+ /// default now that the config default has been flipped), so the
2002+ /// scan still installed a `RowFilter` at read time even though the
2003+ /// parent `FilterExec` was told to keep the predicate. The result was
2004+ /// that the same conjuncts ran twice per batch — once inside the
2005+ /// scan's row filter and once in the surviving `FilterExec` above.
2006+ ///
2007+ /// This test locks in the fix: when the gate declines, the returned
2008+ /// source has `pushdown_filters == false`, guaranteeing the scan
2009+ /// will not install a RowFilter and no double filtering occurs.
2010+ #[ test]
2011+ fn test_narrow_projection_gate_disables_source_pushdown_flag ( ) {
2012+ use arrow:: datatypes:: { DataType , Field , Schema } ;
2013+ use datafusion_common:: config:: ConfigOptions ;
2014+ use datafusion_datasource:: TableSchema ;
2015+ use datafusion_expr:: { col, lit as logical_lit} ;
2016+ use datafusion_physical_expr:: planner:: logical2physical;
2017+ use datafusion_physical_plan:: filter_pushdown:: PushedDown ;
2018+
2019+ // Two-column file schema. Projection covers only column `a`.
2020+ // Filter references column `a`. So:
2021+ // projection = {a} (1 col)
2022+ // filter = {a}
2023+ // non_filter_projected = {a} - {a} = 0
2024+ // 0 < PUSHDOWN_MIN_NON_FILTER_COLS (=3)
2025+ // → gate declines pushdown.
2026+ let file_schema = Arc :: new ( Schema :: new ( vec ! [
2027+ Field :: new( "a" , DataType :: Int64 , false ) ,
2028+ Field :: new( "b" , DataType :: Int64 , false ) ,
2029+ ] ) ) ;
2030+ let table_schema = TableSchema :: from ( file_schema) ;
2031+ let full_schema: arrow:: datatypes:: SchemaRef =
2032+ Arc :: clone ( table_schema. table_schema ( ) ) ;
2033+
2034+ // Narrow projection: only column `a` (index 0).
2035+ let projection = ProjectionExprs :: from_indices ( & [ 0usize ] , & full_schema) ;
2036+ let mut source = ParquetSource :: new ( table_schema) . with_pushdown_filters ( true ) ;
2037+ source. projection = projection;
2038+
2039+ let filter = logical2physical ( & col ( "a" ) . eq ( logical_lit ( 1i64 ) ) , & full_schema) ;
2040+ // Default config keeps `pushdown_filter_mode = Auto`, so the gate
2041+ // is active.
2042+ let config = ConfigOptions :: default ( ) ;
2043+ assert ! ( config. execution. parquet. pushdown_filters) ;
2044+
2045+ let prop = source
2046+ . try_pushdown_filters ( vec ! [ filter] , & config)
2047+ . expect ( "try_pushdown_filters must not error" ) ;
2048+
2049+ // The gate declined: parent must retain the filter.
2050+ assert_eq ! ( prop. filters. len( ) , 1 ) ;
2051+ assert ! (
2052+ matches!( prop. filters[ 0 ] , PushedDown :: No ) ,
2053+ "gate should report PushedDown::No so the FilterExec above the \
2054+ scan keeps the predicate"
2055+ ) ;
2056+
2057+ // Key assertion: the updated source's `pushdown_filters` flag must
2058+ // be `false`. If it were `true`, the scan would install a RowFilter
2059+ // and the same predicate would run twice.
2060+ let updated = prop
2061+ . updated_node
2062+ . as_ref ( )
2063+ . expect ( "gate declined path must attach an updated source" ) ;
2064+ let parquet_src = updated
2065+ . downcast_ref :: < ParquetSource > ( )
2066+ . expect ( "updated node must be a ParquetSource" ) ;
2067+ assert ! (
2068+ !parquet_src. pushdown_filters( ) ,
2069+ "narrow-projection gate must flip source.pushdown_filters to false, \
2070+ otherwise the scan double-filters (RowFilter inside + FilterExec above)"
2071+ ) ;
2072+ }
2073+
2074+ /// A wide projection must NOT trip the gate: pushdown stays enabled
2075+ /// end-to-end (both the return marker and the source-level flag).
2076+ #[ test]
2077+ fn test_wide_projection_keeps_pushdown_enabled ( ) {
2078+ use arrow:: datatypes:: { DataType , Field , Schema } ;
2079+ use datafusion_common:: config:: ConfigOptions ;
2080+ use datafusion_datasource:: TableSchema ;
2081+ use datafusion_expr:: { col, lit as logical_lit} ;
2082+ use datafusion_physical_expr:: planner:: logical2physical;
2083+ use datafusion_physical_plan:: filter_pushdown:: PushedDown ;
2084+
2085+ // Five-column file schema. Projection covers b, c, d, e (4 cols
2086+ // not in the filter set), filter references a only.
2087+ // non_filter_projected = 4 >= PUSHDOWN_MIN_NON_FILTER_COLS
2088+ // → gate does NOT decline.
2089+ let file_schema = Arc :: new ( Schema :: new ( vec ! [
2090+ Field :: new( "a" , DataType :: Int64 , false ) ,
2091+ Field :: new( "b" , DataType :: Int64 , false ) ,
2092+ Field :: new( "c" , DataType :: Int64 , false ) ,
2093+ Field :: new( "d" , DataType :: Int64 , false ) ,
2094+ Field :: new( "e" , DataType :: Int64 , false ) ,
2095+ ] ) ) ;
2096+ let table_schema = TableSchema :: from ( file_schema) ;
2097+ let full_schema: arrow:: datatypes:: SchemaRef =
2098+ Arc :: clone ( table_schema. table_schema ( ) ) ;
2099+ let projection = ProjectionExprs :: from_indices ( & [ 1usize , 2 , 3 , 4 ] , & full_schema) ;
2100+ let mut source = ParquetSource :: new ( table_schema) . with_pushdown_filters ( true ) ;
2101+ source. projection = projection;
2102+
2103+ let filter = logical2physical ( & col ( "a" ) . eq ( logical_lit ( 1i64 ) ) , & full_schema) ;
2104+ let config = ConfigOptions :: default ( ) ;
2105+
2106+ let prop = source
2107+ . try_pushdown_filters ( vec ! [ filter] , & config)
2108+ . expect ( "try_pushdown_filters must not error" ) ;
2109+
2110+ assert_eq ! ( prop. filters. len( ) , 1 ) ;
2111+ assert ! (
2112+ matches!( prop. filters[ 0 ] , PushedDown :: Yes ) ,
2113+ "wide projection should push down"
2114+ ) ;
2115+ let updated = prop
2116+ . updated_node
2117+ . as_ref ( )
2118+ . expect ( "wide projection must attach an updated source too" ) ;
2119+ let parquet_src = updated
2120+ . downcast_ref :: < ParquetSource > ( )
2121+ . expect ( "updated node must be a ParquetSource" ) ;
2122+ assert ! (
2123+ parquet_src. pushdown_filters( ) ,
2124+ "wide projection: source.pushdown_filters must stay true so the \
2125+ scan installs the RowFilter"
2126+ ) ;
2127+ }
2128+
2129+ /// After the narrow-projection gate declines a first pushdown call,
2130+ /// a subsequent call (e.g. TopK injecting a dynamic filter) must be
2131+ /// able to re-enable pushdown. This locks in that the `||` recovery
2132+ /// against the config default keeps the flag reachable for later
2133+ /// callers, so flipping the source flag to `false` in the gate path
2134+ /// is safe.
2135+ #[ test]
2136+ fn test_gate_declined_does_not_permanently_disable_pushdown ( ) {
2137+ use arrow:: datatypes:: { DataType , Field , Schema } ;
2138+ use datafusion_common:: config:: ConfigOptions ;
2139+ use datafusion_datasource:: TableSchema ;
2140+ use datafusion_expr:: { col, lit as logical_lit} ;
2141+ use datafusion_physical_expr:: expressions:: { DynamicFilterPhysicalExpr , lit} ;
2142+ use datafusion_physical_expr:: planner:: logical2physical;
2143+
2144+ let file_schema = Arc :: new ( Schema :: new ( vec ! [
2145+ Field :: new( "a" , DataType :: Int64 , false ) ,
2146+ Field :: new( "b" , DataType :: Int64 , false ) ,
2147+ ] ) ) ;
2148+ let table_schema = TableSchema :: from ( file_schema) ;
2149+ let full_schema: arrow:: datatypes:: SchemaRef =
2150+ Arc :: clone ( table_schema. table_schema ( ) ) ;
2151+ let projection = ProjectionExprs :: from_indices ( & [ 0usize ] , & full_schema) ;
2152+ let mut source = ParquetSource :: new ( table_schema) . with_pushdown_filters ( true ) ;
2153+ source. projection = projection;
2154+
2155+ let filter = logical2physical ( & col ( "a" ) . eq ( logical_lit ( 1i64 ) ) , & full_schema) ;
2156+ let config = ConfigOptions :: default ( ) ;
2157+ let prop = source
2158+ . try_pushdown_filters ( vec ! [ filter] , & config)
2159+ . expect ( "first pushdown call must not error" ) ;
2160+ let after_gate = prop
2161+ . updated_node
2162+ . as_ref ( )
2163+ . expect ( "gate declined path must attach an updated source" )
2164+ . downcast_ref :: < ParquetSource > ( )
2165+ . expect ( "updated node must be a ParquetSource" )
2166+ . clone ( ) ;
2167+ assert ! (
2168+ !after_gate. pushdown_filters( ) ,
2169+ "first call: gate should have flipped source flag to false" ,
2170+ ) ;
2171+
2172+ // Simulate TopK injecting a dynamic filter on the source it just
2173+ // received back from the gate-declined call.
2174+ let dyn_col = Arc :: new ( Column :: new ( "a" , 0 ) ) as Arc < dyn PhysicalExpr > ;
2175+ let dynamic_filter = Arc :: new ( DynamicFilterPhysicalExpr :: new (
2176+ vec ! [ dyn_col] ,
2177+ lit ( true ) as Arc < dyn PhysicalExpr > ,
2178+ ) ) as Arc < dyn PhysicalExpr > ;
2179+
2180+ let prop2 = after_gate
2181+ . try_pushdown_filters ( vec ! [ dynamic_filter] , & config)
2182+ . expect ( "second pushdown call must not error" ) ;
2183+ let after_dynamic = prop2
2184+ . updated_node
2185+ . as_ref ( )
2186+ . expect ( "dynamic-filter path must attach an updated source" )
2187+ . downcast_ref :: < ParquetSource > ( )
2188+ . expect ( "updated node must be a ParquetSource" ) ;
2189+ assert ! (
2190+ after_dynamic. pushdown_filters( ) ,
2191+ "second call with a dynamic filter must re-enable pushdown so the \
2192+ RG-prune / RowFilter cascade can drive the dynamic threshold — \
2193+ the `||` recovery against the config default is what makes this \
2194+ safe after the first call flipped the flag to false"
2195+ ) ;
2196+ }
19862197}
0 commit comments