Skip to content

Commit 1ef3990

Browse files
committed
Address alamb review: add config opt-out, upgrade guide, plan-display marker
Per @alamb's review on #23420: 1. Config opt-out: new `datafusion.execution.parquet.pushdown_filter_narrow_projection_gate` (default true). Set to false to restore the pre-existing unconditional pushdown behavior. Proto + serde support included so the option round-trips through the physical plan. 2. Plan-display marker: when the gate declines pushdown for a scan, the ParquetSource fmt_extra output includes `pushdown_declined=narrow_projection` so profiling / EXPLAIN ANALYZE can see the heuristic fired without needing to correlate config against per-query symptoms. 3. Upgrade guide: docs/source/library-user-guide/upgrading/55.0.0.md documents the new behavior and how to opt out. Kept the heuristic itself unchanged (single hardcoded PUSHDOWN_MIN_NON_FILTER_COLS = 3 with no tuning knob) per @alamb's 'complexity budget' note.
1 parent b6b6421 commit 1ef3990

12 files changed

Lines changed: 102 additions & 10 deletions

File tree

‎datafusion/common/src/config.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1137,6 +1137,15 @@ config_namespace! {
11371137
/// reduce the number of rows decoded. This optimization is sometimes called "late materialization".
11381138
pub pushdown_filters: bool, default = false
11391139

1140+
/// (reading) When `pushdown_filters` is enabled, decline to push filters
1141+
/// into the Parquet scan for queries whose projection contains fewer than
1142+
/// 3 columns not referenced by the filter. RowFilter has a fixed per-row
1143+
/// overhead that only pays for itself when there is enough wide-column
1144+
/// decode to skip; without this gate, narrow-projection queries (e.g.
1145+
/// `SELECT col1 FROM t WHERE col1 <> ''`) regress. Set to `false` to
1146+
/// force unconditional pushdown (the pre-existing behavior).
1147+
pub pushdown_filter_narrow_projection_gate: bool, default = true
1148+
11401149
/// (reading) If true, filter expressions evaluated during the parquet decoding operation
11411150
/// will be reordered heuristically to minimize the cost of evaluation. If false,
11421151
/// the filters are applied in the same order as written in the query

‎datafusion/common/src/file_options/parquet_writer.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,7 @@ impl ParquetOptions {
248248
coerce_int96_tz: _, // not used for writer props
249249
skip_arrow_metadata: _,
250250
max_predicate_cache_size: _,
251+
pushdown_filter_narrow_projection_gate: _, // reads-only, not used for writer props
251252
} = self;
252253

253254
let mut builder = WriterProperties::builder()

‎datafusion/datasource-parquet/src/source.rs‎

Lines changed: 36 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,13 @@ pub struct ParquetSource {
314314
/// Sort order driving `PreparedAccessPlan::reorder_by_statistics`
315315
/// in the opener.
316316
sort_order_for_reorder: Option<LexOrdering>,
317+
/// True when [`try_pushdown_filters`] declined pushdown for this
318+
/// scan due to the narrow-projection gate — surfaced in the plan
319+
/// display so profiling can tell that the heuristic fired here
320+
/// (the filter stays above the scan in a `FilterExec`).
321+
///
322+
/// [`try_pushdown_filters`]: Self::try_pushdown_filters
323+
pub(crate) narrow_projection_gate_declined: bool,
317324
}
318325

319326
impl ParquetSource {
@@ -340,6 +347,7 @@ impl ParquetSource {
340347
encryption_factory: None,
341348
reverse_row_groups: false,
342349
sort_order_for_reorder: None,
350+
narrow_projection_gate_declined: false,
343351
}
344352
}
345353

@@ -740,6 +748,12 @@ impl FileSource for ParquetSource {
740748
if self.reverse_row_groups {
741749
write!(f, ", reverse_row_groups=true")?;
742750
}
751+
if self.narrow_projection_gate_declined {
752+
write!(
753+
f,
754+
", pushdown_declined=narrow_projection"
755+
)?;
756+
}
743757

744758
// Plan-time marker for dynamic RG-level pruning: if the
745759
// predicate is dynamic (e.g. a TopK threshold expression),
@@ -830,24 +844,34 @@ impl FileSource for ParquetSource {
830844
let table_pushdown_enabled = self.pushdown_filters();
831845
let mut pushdown_filters = table_pushdown_enabled || config_pushdown_enabled;
832846

833-
// Simple heuristic (issue #3463, discussion on PR #23369):
847+
// Narrow-projection gate (issue #3463, discussion on PR #23369):
834848
// RowFilter has a fixed per-row machinery overhead that only pays
835849
// for itself when the wide-column decode it lets us skip is
836850
// meaningful. When the projection is narrow and the filter columns
837851
// already cover most of it (typical for `GROUP BY col`-style
838852
// queries such as ClickBench Q10/Q11/Q40), the overhead dominates
839-
// and pushdown regresses. Decline pushdown for those scans; keep it
840-
// for wider projections where the fast-path pays.
853+
// and pushdown regresses. Decline pushdown for those scans; keep
854+
// it for wider projections where the fast-path pays.
855+
//
856+
// Kept intentionally simple: a plan-time column-count check
857+
// gated on a single boolean opt-out. A future
858+
// adaptive-placement pass ([#22883]) can supersede this with a
859+
// runtime cost model.
841860
//
842-
// Kept intentionally simple: a plan-time column-count check with
843-
// no tuning knob. A future adaptive-placement pass (#22883) can
844-
// supersede this with a runtime cost model.
861+
// When declined: the filter stays above the scan in a
862+
// `FilterExec` (correctness preserved) and the predicate is
863+
// still injected into `ParquetSource` for stats / bloom /
864+
// page-index pruning.
845865
//
846-
// When declined: the filter stays above the scan in a FilterExec
847-
// (correctness preserved) and the predicate is still injected
848-
// into `ParquetSource` for stats / bloom / page-index pruning.
866+
// Users who want the pre-existing "always push" behavior can
867+
// set `pushdown_filter_narrow_projection_gate = false`.
868+
//
869+
// [#22883]: https://github.com/apache/datafusion/issues/22883
849870
const PUSHDOWN_MIN_NON_FILTER_COLS: usize = 3;
850-
if pushdown_filters {
871+
let mut narrow_projection_gate_declined = false;
872+
if pushdown_filters
873+
&& config.execution.parquet.pushdown_filter_narrow_projection_gate
874+
{
851875
let filter_col_indices: std::collections::HashSet<usize> = filters
852876
.iter()
853877
.flat_map(|f| collect_columns(f).into_iter().map(|c| c.index()))
@@ -863,10 +887,12 @@ impl FileSource for ParquetSource {
863887
.len();
864888
if non_filter_projected < PUSHDOWN_MIN_NON_FILTER_COLS {
865889
pushdown_filters = false;
890+
narrow_projection_gate_declined = true;
866891
}
867892
}
868893

869894
let mut source = self.clone();
895+
source.narrow_projection_gate_declined = narrow_projection_gate_declined;
870896
let filters: Vec<PushedDownPredicate> = filters
871897
.into_iter()
872898
.map(|filter| {

‎datafusion/proto-common/proto/datafusion_common.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -631,6 +631,8 @@ message ParquetOptions {
631631
uint64 max_row_group_bytes = 37;
632632
}
633633

634+
bool pushdown_filter_narrow_projection_gate = 38;
635+
634636
ParquetCdcOptions content_defined_chunking = 35;
635637

636638
// Optional timezone applied to INT96-coerced timestamps when `coerce_int96`

‎datafusion/proto-common/src/from_proto/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1133,6 +1133,7 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
11331133
max_row_group_bytes: value.max_row_group_bytes_opt.and_then(|opt| match opt {
11341134
protobuf::parquet_options::MaxRowGroupBytesOpt::MaxRowGroupBytes(v) => MaxRowGroupBytes::try_new(v as usize).ok(),
11351135
}),
1136+
pushdown_filter_narrow_projection_gate: value.pushdown_filter_narrow_projection_gate,
11361137
content_defined_chunking: value.content_defined_chunking.map(ParquetCdcOptions::from).unwrap_or_default(),
11371138
})
11381139
}

‎datafusion/proto-common/src/generated/pbjson.rs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6412,6 +6412,9 @@ impl serde::Serialize for ParquetOptions {
64126412
if !self.created_by.is_empty() {
64136413
len += 1;
64146414
}
6415+
if self.pushdown_filter_narrow_projection_gate {
6416+
len += 1;
6417+
}
64156418
if self.content_defined_chunking.is_some() {
64166419
len += 1;
64176420
}
@@ -6532,6 +6535,9 @@ impl serde::Serialize for ParquetOptions {
65326535
if !self.created_by.is_empty() {
65336536
struct_ser.serialize_field("createdBy", &self.created_by)?;
65346537
}
6538+
if self.pushdown_filter_narrow_projection_gate {
6539+
struct_ser.serialize_field("pushdownFilterNarrowProjectionGate", &self.pushdown_filter_narrow_projection_gate)?;
6540+
}
65356541
if let Some(v) = self.content_defined_chunking.as_ref() {
65366542
struct_ser.serialize_field("contentDefinedChunking", v)?;
65376543
}
@@ -6689,6 +6695,8 @@ impl<'de> serde::Deserialize<'de> for ParquetOptions {
66896695
"maxRowGroupSize",
66906696
"created_by",
66916697
"createdBy",
6698+
"pushdown_filter_narrow_projection_gate",
6699+
"pushdownFilterNarrowProjectionGate",
66926700
"content_defined_chunking",
66936701
"contentDefinedChunking",
66946702
"metadata_size_hint",
@@ -6740,6 +6748,7 @@ impl<'de> serde::Deserialize<'de> for ParquetOptions {
67406748
DataPageRowCountLimit,
67416749
MaxRowGroupSize,
67426750
CreatedBy,
6751+
PushdownFilterNarrowProjectionGate,
67436752
ContentDefinedChunking,
67446753
MetadataSizeHint,
67456754
Compression,
@@ -6796,6 +6805,7 @@ impl<'de> serde::Deserialize<'de> for ParquetOptions {
67966805
"dataPageRowCountLimit" | "data_page_row_count_limit" => Ok(GeneratedField::DataPageRowCountLimit),
67976806
"maxRowGroupSize" | "max_row_group_size" => Ok(GeneratedField::MaxRowGroupSize),
67986807
"createdBy" | "created_by" => Ok(GeneratedField::CreatedBy),
6808+
"pushdownFilterNarrowProjectionGate" | "pushdown_filter_narrow_projection_gate" => Ok(GeneratedField::PushdownFilterNarrowProjectionGate),
67996809
"contentDefinedChunking" | "content_defined_chunking" => Ok(GeneratedField::ContentDefinedChunking),
68006810
"metadataSizeHint" | "metadata_size_hint" => Ok(GeneratedField::MetadataSizeHint),
68016811
"compression" => Ok(GeneratedField::Compression),
@@ -6850,6 +6860,7 @@ impl<'de> serde::Deserialize<'de> for ParquetOptions {
68506860
let mut data_page_row_count_limit__ = None;
68516861
let mut max_row_group_size__ = None;
68526862
let mut created_by__ = None;
6863+
let mut pushdown_filter_narrow_projection_gate__ = None;
68536864
let mut content_defined_chunking__ = None;
68546865
let mut metadata_size_hint_opt__ = None;
68556866
let mut compression_opt__ = None;
@@ -7006,6 +7017,12 @@ impl<'de> serde::Deserialize<'de> for ParquetOptions {
70067017
}
70077018
created_by__ = Some(map_.next_value()?);
70087019
}
7020+
GeneratedField::PushdownFilterNarrowProjectionGate => {
7021+
if pushdown_filter_narrow_projection_gate__.is_some() {
7022+
return Err(serde::de::Error::duplicate_field("pushdownFilterNarrowProjectionGate"));
7023+
}
7024+
pushdown_filter_narrow_projection_gate__ = Some(map_.next_value()?);
7025+
}
70097026
GeneratedField::ContentDefinedChunking => {
70107027
if content_defined_chunking__.is_some() {
70117028
return Err(serde::de::Error::duplicate_field("contentDefinedChunking"));
@@ -7114,6 +7131,7 @@ impl<'de> serde::Deserialize<'de> for ParquetOptions {
71147131
data_page_row_count_limit: data_page_row_count_limit__.unwrap_or_default(),
71157132
max_row_group_size: max_row_group_size__.unwrap_or_default(),
71167133
created_by: created_by__.unwrap_or_default(),
7134+
pushdown_filter_narrow_projection_gate: pushdown_filter_narrow_projection_gate__.unwrap_or_default(),
71177135
content_defined_chunking: content_defined_chunking__,
71187136
metadata_size_hint_opt: metadata_size_hint_opt__,
71197137
compression_opt: compression_opt__,

‎datafusion/proto-common/src/generated/prost.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -864,6 +864,8 @@ pub struct ParquetOptions {
864864
pub max_row_group_size: u64,
865865
#[prost(string, tag = "16")]
866866
pub created_by: ::prost::alloc::string::String,
867+
#[prost(bool, tag = "38")]
868+
pub pushdown_filter_narrow_projection_gate: bool,
867869
#[prost(message, optional, tag = "35")]
868870
pub content_defined_chunking: ::core::option::Option<ParquetCdcOptions>,
869871
#[prost(oneof = "parquet_options::MetadataSizeHintOpt", tags = "4")]

‎datafusion/proto-common/src/to_proto/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -939,6 +939,7 @@ impl TryFrom<&ParquetOptions> for protobuf::ParquetOptions {
939939
coerce_int96_tz_opt: value.coerce_int96_tz.clone().map(protobuf::parquet_options::CoerceInt96TzOpt::CoerceInt96Tz),
940940
max_predicate_cache_size_opt: value.max_predicate_cache_size.map(|v| protobuf::parquet_options::MaxPredicateCacheSizeOpt::MaxPredicateCacheSize(v as u64)),
941941
max_row_group_bytes_opt: value.max_row_group_bytes.map(|v| protobuf::parquet_options::MaxRowGroupBytesOpt::MaxRowGroupBytes(v.get() as u64)),
942+
pushdown_filter_narrow_projection_gate: value.pushdown_filter_narrow_projection_gate,
942943
content_defined_chunking: Some((&value.content_defined_chunking).into()),
943944
})
944945
}

‎datafusion/proto-models/src/generated/datafusion_proto_common.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -864,6 +864,8 @@ pub struct ParquetOptions {
864864
pub max_row_group_size: u64,
865865
#[prost(string, tag = "16")]
866866
pub created_by: ::prost::alloc::string::String,
867+
#[prost(bool, tag = "38")]
868+
pub pushdown_filter_narrow_projection_gate: bool,
867869
#[prost(message, optional, tag = "35")]
868870
pub content_defined_chunking: ::core::option::Option<ParquetCdcOptions>,
869871
#[prost(oneof = "parquet_options::MetadataSizeHintOpt", tags = "4")]

‎datafusion/proto/src/logical_plan/file_formats.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,6 +461,7 @@ mod parquet {
461461
max_row_group_bytes_opt: global_options.global.max_row_group_bytes.map(|size| {
462462
parquet_options::MaxRowGroupBytesOpt::MaxRowGroupBytes(size.get() as u64)
463463
}),
464+
pushdown_filter_narrow_projection_gate: global_options.global.pushdown_filter_narrow_projection_gate,
464465
content_defined_chunking: Some(ParquetCdcOptionsProto {
465466
enabled: global_options.global.content_defined_chunking.enabled,
466467
min_chunk_size: global_options.global.content_defined_chunking.min_chunk_size as u64,
@@ -642,6 +643,8 @@ mod parquet {
642643
MaxRowGroupBytes::try_new(*size as usize).ok()
643644
}
644645
}),
646+
pushdown_filter_narrow_projection_gate: proto
647+
.pushdown_filter_narrow_projection_gate,
645648
content_defined_chunking: proto
646649
.content_defined_chunking
647650
.map(ParquetCdcOptions::from_proto)

0 commit comments

Comments
 (0)