@@ -461,11 +461,10 @@ struct PreparedParquetOpen {
461461 enable_page_index : bool ,
462462 enable_bloom_filter : bool ,
463463 enable_row_group_stats_pruning : bool ,
464- /// True when the (original) predicate references a column that
465- /// `constant_columns_from_stats` proved constant for this file, so a
466- /// predicate that later simplifies to a constant did so on the
467- /// strength of file statistics — see `prune_row_groups`.
468- stats_constants_in_predicate : bool ,
464+ /// True when substituting the columns `constant_columns_from_stats`
465+ /// proved constant for this file is on its own enough to collapse the
466+ /// predicate to a constant no row can satisfy — see `prune_row_groups`.
467+ stats_prove_unsatisfiable : bool ,
469468 limit : Option < usize > ,
470469 coerce_int96 : Option < TimeUnit > ,
471470 coerce_int96_tz : Option < Arc < str > > ,
@@ -804,11 +803,23 @@ impl ParquetMorselizer {
804803 partitioned_file. statistics . as_deref ( ) ,
805804 & logical_file_schema,
806805 ) ;
807- let stats_constants_in_predicate = !stats_constants. is_empty ( )
806+ // Whether the statistics substitution *itself* proves the predicate
807+ // unsatisfiable has to be decided here, before the partition-value
808+ // and missing-column rewriting below: a predicate can collapse to a
809+ // constant through either of those too, and only a collapse the
810+ // file's own statistics produced is proof for `prune_row_groups`.
811+ // `collect_columns` is the cheap short-circuit so the simplifier
812+ // only runs when a stats-derived constant is actually referenced.
813+ let stats_prove_unsatisfiable = !stats_constants. is_empty ( )
808814 && self . predicate . as_ref ( ) . is_some_and ( |p| {
809815 collect_columns ( p)
810816 . iter ( )
811817 . any ( |c| stats_constants. contains_key ( c. name ( ) ) )
818+ && stats_alone_unsatisfiable (
819+ p,
820+ & stats_constants,
821+ self . table_schema . table_schema ( ) ,
822+ )
812823 } ) ;
813824 literal_columns. extend ( stats_constants) ;
814825
@@ -880,7 +891,7 @@ impl ParquetMorselizer {
880891 enable_page_index : self . enable_page_index ,
881892 enable_bloom_filter : self . enable_bloom_filter ,
882893 enable_row_group_stats_pruning : self . enable_row_group_stats_pruning ,
883- stats_constants_in_predicate ,
894+ stats_prove_unsatisfiable ,
884895 limit : self . limit ,
885896 coerce_int96 : self . coerce_int96 ,
886897 coerce_int96_tz : self . coerce_int96_tz . clone ( ) ,
@@ -1142,33 +1153,26 @@ impl FiltersPreparedParquetOpen {
11421153 row_groups. prune_by_range ( rg_metadata, range) ;
11431154 }
11441155
1145- // If constant-column substitution (`constant_columns_from_stats`)
1146- // plus simplification collapsed the predicate to a constant that can
1147- // never be true (`false`, or NULL — filters treat NULL as false),
1148- // the file's own statistics have proven that no row can match: skip
1149- // every remaining row group, credited to statistics pruning. Without
1150- // this, the substitution is strictly counterproductive for such
1151- // files: it *removes* the column references the pruning predicate
1152- // would need (`build_pruning_predicates` returns `None` for a bare
1153- // literal), so a file that was previously pruned via its
1154- // `null_count` statistics is instead scanned in full.
1156+ // Substituting the columns file statistics proved constant
1157+ // (`constant_columns_from_stats`) collapsed the predicate to a
1158+ // constant that can never be true (`false`, or NULL — filters treat
1159+ // NULL as false), so the file's own statistics have proven that no
1160+ // row can match: skip every remaining row group, credited to
1161+ // statistics pruning. Without this, the substitution is strictly
1162+ // counterproductive for such files: it *removes* the column
1163+ // references the pruning predicate would need
1164+ // (`build_pruning_predicates` returns `None` for a bare literal), so
1165+ // a file that was previously pruned via its `null_count` statistics
1166+ // is instead scanned in full.
11551167 //
1156- // Deliberately narrowed by `stats_constants_in_predicate`: a
1157- // predicate can also collapse via the missing-column adapter or
1158- // partition-value folding, and those paths keep their existing
1159- // behaviour (files pruned by partition values are `FilePruner`'s
1160- // job). Only a collapse the file's own statistics contributed to
1161- // is treated as proof here.
1162- if prepared. enable_row_group_stats_pruning
1163- && prepared. stats_constants_in_predicate
1164- && prepared
1165- . predicate
1166- . as_ref ( )
1167- . and_then ( |p| p. downcast_ref :: < Literal > ( ) )
1168- . is_some_and ( |l| {
1169- l. value ( ) . is_null ( ) || l. value ( ) == & ScalarValue :: Boolean ( Some ( false ) )
1170- } )
1171- {
1168+ // The flag is deliberately computed from the statistics substitution
1169+ // alone (see `ParquetMorselizer::prepare`). A predicate can also
1170+ // collapse via the missing-column adapter or partition-value
1171+ // folding, and those paths keep their existing behaviour (files
1172+ // pruned by partition values are `FilePruner`'s job); a mixed
1173+ // predicate that only collapses once one of them has run is not
1174+ // treated as proof here.
1175+ if prepared. enable_row_group_stats_pruning && prepared. stats_prove_unsatisfiable {
11721176 prepared
11731177 . file_metrics
11741178 . row_groups_pruned_statistics
@@ -1761,6 +1765,38 @@ fn row_group_bytes(rg_meta: &RowGroupMetaData) -> u64 {
17611765
17621766type ConstantColumns = HashMap < String , ScalarValue > ;
17631767
1768+ /// True when substituting `stats_constants` — the columns this file's
1769+ /// statistics prove constant — is on its own enough to collapse `predicate`
1770+ /// to a constant no row can satisfy (`false`, or NULL, which filters treat
1771+ /// as false).
1772+ ///
1773+ /// Only the statistics substitution is applied, so a predicate that needs
1774+ /// the missing-column adapter or partition-value folding to collapse does
1775+ /// not qualify: those are separate proofs with their own pruning paths.
1776+ ///
1777+ /// `schema` must resolve every column the predicate can reference — pass the
1778+ /// full table schema, since the simplifier types each node as it walks and
1779+ /// the substitution leaves non-constant columns in place. Substitution or
1780+ /// simplification failing is not proof, so both answer `false` and leave the
1781+ /// caller's normal pruning paths untouched.
1782+ fn stats_alone_unsatisfiable (
1783+ predicate : & Arc < dyn PhysicalExpr > ,
1784+ stats_constants : & ConstantColumns ,
1785+ schema : & Schema ,
1786+ ) -> bool {
1787+ let Ok ( substituted) =
1788+ replace_columns_with_literals ( Arc :: clone ( predicate) , stats_constants)
1789+ else {
1790+ return false ;
1791+ } ;
1792+ let Ok ( simplified) = PhysicalExprSimplifier :: new ( schema) . simplify ( substituted) else {
1793+ return false ;
1794+ } ;
1795+ simplified. downcast_ref :: < Literal > ( ) . is_some_and ( |l| {
1796+ l. value ( ) . is_null ( ) || l. value ( ) == & ScalarValue :: Boolean ( Some ( false ) )
1797+ } )
1798+ }
1799+
17641800/// Extract constant column values from statistics, keyed by column name in the logical file schema.
17651801fn constant_columns_from_stats (
17661802 statistics : Option < & Statistics > ,
@@ -3075,6 +3111,28 @@ mod test {
30753111 assert_eq ! ( num_rows, 0 ) ;
30763112 }
30773113
3114+ /// Row groups this scan skipped through statistics pruning.
3115+ ///
3116+ /// The row count alone cannot tell "pruned" from "scanned, then
3117+ /// row-filtered" — both yield zero rows — so the tests around
3118+ /// constant-column substitution assert on this metric instead.
3119+ fn pruned_row_groups_statistics ( metrics : & ExecutionPlanMetricsSet ) -> usize {
3120+ use datafusion_physical_plan:: metrics:: MetricValue ;
3121+ metrics
3122+ . clone_inner ( )
3123+ . iter ( )
3124+ . find_map ( |m| match m. value ( ) {
3125+ MetricValue :: PruningMetrics {
3126+ name,
3127+ pruning_metrics,
3128+ } if name == "row_groups_pruned_statistics" => {
3129+ Some ( pruning_metrics. pruned ( ) )
3130+ }
3131+ _ => None ,
3132+ } )
3133+ . expect ( "row_groups_pruned_statistics metric is emitted" )
3134+ }
3135+
30783136 #[ tokio:: test]
30793137 async fn test_prune_all_null_column_equality_from_file_statistics ( ) {
30803138 // Regression: a column whose file statistics say every value is
@@ -3111,25 +3169,6 @@ mod test {
31113169 Field :: new( "b" , DataType :: Utf8 , true ) ,
31123170 ] ) ) ;
31133171 let table_schema_for_opener = TableSchemaBuilder :: from ( & file_schema) . build ( ) ;
3114- // The row count alone cannot tell "pruned" from "scanned, then
3115- // row-filtered" — both yield zero rows here — so assert on the
3116- // pruning metric, which is the behaviour under test.
3117- use datafusion_physical_plan:: metrics:: MetricValue ;
3118- let pruned_row_groups = |metrics : & ExecutionPlanMetricsSet | {
3119- metrics
3120- . clone_inner ( )
3121- . iter ( )
3122- . find_map ( |m| match m. value ( ) {
3123- MetricValue :: PruningMetrics {
3124- name,
3125- pruning_metrics,
3126- } if name == "row_groups_pruned_statistics" => {
3127- Some ( pruning_metrics. pruned ( ) )
3128- }
3129- _ => None ,
3130- } )
3131- . expect ( "row_groups_pruned_statistics metric is emitted" )
3132- } ;
31333172 let make_opener = |predicate, metrics : ExecutionPlanMetricsSet | {
31343173 ParquetMorselizerBuilder :: new ( )
31353174 . with_store ( Arc :: clone ( & store) )
@@ -3151,7 +3190,7 @@ mod test {
31513190 let ( _, num_rows) = count_batches_and_rows ( stream) . await ;
31523191 assert_eq ! ( num_rows, 0 ) ;
31533192 assert_eq ! (
3154- pruned_row_groups ( & metrics) ,
3193+ pruned_row_groups_statistics ( & metrics) ,
31553194 1 ,
31563195 "an all-NULL column cannot satisfy equality: the row group must be pruned"
31573196 ) ;
@@ -3166,7 +3205,122 @@ mod test {
31663205 let ( num_batches, num_rows) = count_batches_and_rows ( stream) . await ;
31673206 assert_eq ! ( num_batches, 1 ) ;
31683207 assert_eq ! ( num_rows, 3 ) ;
3169- assert_eq ! ( pruned_row_groups( & metrics) , 0 ) ;
3208+ assert_eq ! ( pruned_row_groups_statistics( & metrics) , 0 ) ;
3209+ }
3210+
3211+ #[ tokio:: test]
3212+ async fn test_prune_exact_constant_column_false_predicate_from_file_statistics ( ) {
3213+ // The `false` half of the branch above, which the all-NULL case
3214+ // only covers for NULL: exact min == max statistics prove `a`
3215+ // constant, so `a = <other literal>` folds to `false` and the row
3216+ // group is pruned on that proof.
3217+ let store = Arc :: new ( InMemory :: new ( ) ) as Arc < dyn ObjectStore > ;
3218+
3219+ let batch = record_batch ! ( ( "a" , Int32 , vec![ Some ( 7 ) , Some ( 7 ) , Some ( 7 ) ] ) ) . unwrap ( ) ;
3220+ let data_size =
3221+ write_parquet ( Arc :: clone ( & store) , "file.parquet" , batch. clone ( ) ) . await ;
3222+ let file_schema = batch. schema ( ) ;
3223+ let mut file = PartitionedFile :: new (
3224+ "file.parquet" . to_string ( ) ,
3225+ u64:: try_from ( data_size) . unwrap ( ) ,
3226+ ) ;
3227+ let mut statistics = Statistics :: new_unknown ( & file_schema) ;
3228+ statistics. num_rows = Precision :: Exact ( 3 ) ;
3229+ statistics. column_statistics [ 0 ] . null_count = Precision :: Exact ( 0 ) ;
3230+ statistics. column_statistics [ 0 ] . min_value =
3231+ Precision :: Exact ( ScalarValue :: Int32 ( Some ( 7 ) ) ) ;
3232+ statistics. column_statistics [ 0 ] . max_value =
3233+ Precision :: Exact ( ScalarValue :: Int32 ( Some ( 7 ) ) ) ;
3234+ file. statistics = Some ( Arc :: new ( statistics) ) ;
3235+ let table_schema =
3236+ Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , false ) ] ) ) ;
3237+ let table_schema_for_opener = TableSchemaBuilder :: from ( & file_schema) . build ( ) ;
3238+
3239+ let metrics = ExecutionPlanMetricsSet :: new ( ) ;
3240+ let expr = col ( "a" ) . eq ( lit ( 8 ) ) ;
3241+ let predicate = logical2physical ( & expr, & table_schema) ;
3242+ let opener = ParquetMorselizerBuilder :: new ( )
3243+ . with_store ( Arc :: clone ( & store) )
3244+ . with_table_schema ( table_schema_for_opener)
3245+ . with_projection_indices ( & [ 0 ] )
3246+ . with_predicate ( predicate)
3247+ . with_row_group_stats_pruning ( true )
3248+ . with_metrics ( metrics. clone ( ) )
3249+ . build ( ) ;
3250+ let stream = open_file ( & opener, file) . await . unwrap ( ) ;
3251+ let ( _, num_rows) = count_batches_and_rows ( stream) . await ;
3252+ assert_eq ! ( num_rows, 0 ) ;
3253+ assert_eq ! (
3254+ pruned_row_groups_statistics( & metrics) ,
3255+ 1 ,
3256+ "a column statistics prove constant cannot equal a different literal: \
3257+ the row group must be pruned"
3258+ ) ;
3259+ }
3260+
3261+ #[ tokio:: test]
3262+ async fn test_no_prune_when_missing_column_collapses_mixed_predicate ( ) {
3263+ // A stats-derived constant appearing in the predicate is not proof
3264+ // that the statistics are what made it unsatisfiable. Here `a = 1`
3265+ // is proven true by statistics and folds away, and the collapse to
3266+ // NULL comes entirely from the missing-column adapter folding
3267+ // `b = 2`. That is the schema-evolution path, which deliberately
3268+ // scans and row-filters rather than pruning, so the statistics
3269+ // branch must not fire.
3270+ let store = Arc :: new ( InMemory :: new ( ) ) as Arc < dyn ObjectStore > ;
3271+
3272+ // The file has only `a`; `b` is in the table schema but missing
3273+ // from this file.
3274+ let batch = record_batch ! ( ( "a" , Int32 , vec![ Some ( 1 ) , Some ( 1 ) , Some ( 1 ) ] ) ) . unwrap ( ) ;
3275+ let data_size =
3276+ write_parquet ( Arc :: clone ( & store) , "file.parquet" , batch. clone ( ) ) . await ;
3277+ let physical_schema = batch. schema ( ) ;
3278+ let mut file = PartitionedFile :: new (
3279+ "file.parquet" . to_string ( ) ,
3280+ u64:: try_from ( data_size) . unwrap ( ) ,
3281+ ) ;
3282+ let logical_file_schema = Arc :: new ( Schema :: new ( vec ! [
3283+ Field :: new( "a" , DataType :: Int32 , false ) ,
3284+ Field :: new( "b" , DataType :: Int32 , true ) ,
3285+ ] ) ) ;
3286+ let mut statistics = Statistics :: new_unknown ( & logical_file_schema) ;
3287+ statistics. num_rows = Precision :: Exact ( 3 ) ;
3288+ statistics. column_statistics [ 0 ] . null_count = Precision :: Exact ( 0 ) ;
3289+ statistics. column_statistics [ 0 ] . min_value =
3290+ Precision :: Exact ( ScalarValue :: Int32 ( Some ( 1 ) ) ) ;
3291+ statistics. column_statistics [ 0 ] . max_value =
3292+ Precision :: Exact ( ScalarValue :: Int32 ( Some ( 1 ) ) ) ;
3293+ file. statistics = Some ( Arc :: new ( statistics) ) ;
3294+ let table_schema_for_opener =
3295+ TableSchemaBuilder :: from ( & logical_file_schema) . build ( ) ;
3296+
3297+ let metrics = ExecutionPlanMetricsSet :: new ( ) ;
3298+ let expr = col ( "a" ) . eq ( lit ( 1 ) ) . and ( col ( "b" ) . eq ( lit ( 2 ) ) ) ;
3299+ let predicate = logical2physical ( & expr, & logical_file_schema) ;
3300+ let opener = ParquetMorselizerBuilder :: new ( )
3301+ . with_store ( Arc :: clone ( & store) )
3302+ . with_table_schema ( table_schema_for_opener)
3303+ . with_projection_indices ( & [ 0 ] )
3304+ . with_predicate ( predicate)
3305+ . with_row_group_stats_pruning ( true )
3306+ . with_metrics ( metrics. clone ( ) )
3307+ . build ( ) ;
3308+ let stream = open_file ( & opener, file) . await . unwrap ( ) ;
3309+ let ( _, num_rows) = count_batches_and_rows ( stream) . await ;
3310+ // The row group is read and its rows handed up for the filter above
3311+ // to apply, which is the missing-column path this test protects.
3312+ assert_eq ! (
3313+ num_rows, 3 ,
3314+ "the file must be scanned, not pruned, on the missing-column path"
3315+ ) ;
3316+ assert_eq ! (
3317+ pruned_row_groups_statistics( & metrics) ,
3318+ 0 ,
3319+ "the collapse came from the missing column, not from file \
3320+ statistics: the statistics branch must not claim the prune"
3321+ ) ;
3322+ // Guard against the file schema drifting: `b` really is absent.
3323+ assert ! ( physical_schema. field_with_name( "b" ) . is_err( ) ) ;
31703324 }
31713325
31723326 #[ tokio:: test]
0 commit comments