@@ -52,6 +52,11 @@ use std::any::Any;
5252use std:: collections:: HashMap ;
5353use std:: sync:: Arc ;
5454
55+ /// Minimum fraction of row groups that must report NDV statistics for the
56+ /// merged result to be `Inexact` rather than `Absent`, as the estimate
57+ /// would be too unreliable otherwise.
58+ const PARTIAL_NDV_THRESHOLD : f64 = 0.75 ;
59+
5560/// Handles fetching Parquet file schema, metadata and statistics
5661/// from object store.
5762///
@@ -544,6 +549,7 @@ fn summarize_column_statistics(
544549 // Extract distinct counts from row group column statistics
545550 accumulators. distinct_counts_array [ logical_schema_index] =
546551 if let Some ( parquet_idx) = parquet_index {
552+ let num_row_groups = row_groups_metadata. len ( ) ;
547553 let distinct_counts: Vec < u64 > = row_groups_metadata
548554 . iter ( )
549555 . filter_map ( |rg| {
@@ -554,9 +560,12 @@ fn summarize_column_statistics(
554560 } )
555561 . collect ( ) ;
556562
557- if distinct_counts. is_empty ( ) {
563+ let coverage =
564+ distinct_counts. len ( ) as f64 / num_row_groups. max ( 1 ) as f64 ;
565+
566+ if coverage < PARTIAL_NDV_THRESHOLD {
558567 Precision :: Absent
559- } else if distinct_counts. len ( ) == 1 {
568+ } else if distinct_counts. len ( ) == 1 && num_row_groups == 1 {
560569 // Single row group with distinct count - use exact value
561570 Precision :: Exact ( distinct_counts[ 0 ] as usize )
562571 } else {
@@ -1029,20 +1038,17 @@ mod tests {
10291038 }
10301039
10311040 #[ test]
1032- fn test_distinct_count_partial_ndv_in_row_groups ( ) {
1033- // Some row groups have NDV, some don't - should use only those that have it
1041+ fn test_distinct_count_partial_ndv_below_threshold ( ) {
1042+ // 1 of 2 row groups has NDV (50% < 75% threshold) -> Absent
10341043 let schema_descr = create_schema_descr ( 1 ) ;
10351044 let arrow_schema = create_arrow_schema ( 1 ) ;
10361045
1037- // Row group 1: has distinct_count = 15
10381046 let stats1 =
10391047 ParquetStatistics :: int32 ( Some ( 1 ) , Some ( 50 ) , Some ( 15 ) , Some ( 0 ) , false ) ;
1040-
1041- // Row group 2: no distinct_count
10421048 let stats2 = ParquetStatistics :: int32 (
10431049 Some ( 51 ) ,
10441050 Some ( 100 ) ,
1045- None , // no distinct_count
1051+ None ,
10461052 Some ( 0 ) ,
10471053 false ,
10481054 ) ;
@@ -1060,10 +1066,39 @@ mod tests {
10601066 )
10611067 . unwrap ( ) ;
10621068
1063- // Only one row group has NDV, so it's Exact(15)
10641069 assert_eq ! (
10651070 result. column_statistics[ 0 ] . distinct_count,
1066- Precision :: Exact ( 15 )
1071+ Precision :: Absent
1072+ ) ;
1073+ }
1074+
1075+ #[ test]
1076+ fn test_distinct_count_partial_ndv_above_threshold ( ) {
1077+ // 3 of 4 row groups have NDV (75% >= 75% threshold) -> Inexact
1078+ let schema_descr = create_schema_descr ( 1 ) ;
1079+ let arrow_schema = create_arrow_schema ( 1 ) ;
1080+
1081+ let stats_with =
1082+ |ndv| ParquetStatistics :: int32 ( Some ( 1 ) , Some ( 100 ) , Some ( ndv) , Some ( 0 ) , false ) ;
1083+ let stats_without =
1084+ ParquetStatistics :: int32 ( Some ( 1 ) , Some ( 100 ) , None , Some ( 0 ) , false ) ;
1085+
1086+ let rg1 = create_row_group_with_stats ( & schema_descr, vec ! [ Some ( stats_with( 10 ) ) ] , 250 ) ;
1087+ let rg2 = create_row_group_with_stats ( & schema_descr, vec ! [ Some ( stats_with( 20 ) ) ] , 250 ) ;
1088+ let rg3 = create_row_group_with_stats ( & schema_descr, vec ! [ Some ( stats_with( 15 ) ) ] , 250 ) ;
1089+ let rg4 = create_row_group_with_stats ( & schema_descr, vec ! [ Some ( stats_without) ] , 250 ) ;
1090+ let metadata =
1091+ create_parquet_metadata ( schema_descr, vec ! [ rg1, rg2, rg3, rg4] ) ;
1092+
1093+ let result = DFParquetMetadata :: statistics_from_parquet_metadata (
1094+ & metadata,
1095+ & arrow_schema,
1096+ )
1097+ . unwrap ( ) ;
1098+
1099+ assert_eq ! (
1100+ result. column_statistics[ 0 ] . distinct_count,
1101+ Precision :: Inexact ( 20 )
10671102 ) ;
10681103 }
10691104
0 commit comments