Skip to content

Commit faf5211

Browse files
committed
Add threshold for partial NDV extraction from Parquet row groups
When only a fraction of row groups report NDV statistics, the merged estimate can be misleading. Require at least 75% of row groups to have NDV before reporting an Inexact value, otherwise return Absent.
1 parent 72d3be3 commit faf5211

1 file changed

Lines changed: 45 additions & 10 deletions

File tree

‎datafusion/datasource-parquet/src/metadata.rs‎

Lines changed: 45 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@ use std::any::Any;
5252
use std::collections::HashMap;
5353
use std::sync::Arc;
5454

55+
/// Minimum fraction of row groups that must report NDV statistics for the
56+
/// merged result to be `Inexact` rather than `Absent`, as the estimate
57+
/// would be too unreliable otherwise.
58+
const PARTIAL_NDV_THRESHOLD: f64 = 0.75;
59+
5560
/// Handles fetching Parquet file schema, metadata and statistics
5661
/// from object store.
5762
///
@@ -544,6 +549,7 @@ fn summarize_column_statistics(
544549
// Extract distinct counts from row group column statistics
545550
accumulators.distinct_counts_array[logical_schema_index] =
546551
if let Some(parquet_idx) = parquet_index {
552+
let num_row_groups = row_groups_metadata.len();
547553
let distinct_counts: Vec<u64> = row_groups_metadata
548554
.iter()
549555
.filter_map(|rg| {
@@ -554,9 +560,12 @@ fn summarize_column_statistics(
554560
})
555561
.collect();
556562

557-
if distinct_counts.is_empty() {
563+
let coverage =
564+
distinct_counts.len() as f64 / num_row_groups.max(1) as f64;
565+
566+
if coverage < PARTIAL_NDV_THRESHOLD {
558567
Precision::Absent
559-
} else if distinct_counts.len() == 1 {
568+
} else if distinct_counts.len() == 1 && num_row_groups == 1 {
560569
// Single row group with distinct count - use exact value
561570
Precision::Exact(distinct_counts[0] as usize)
562571
} else {
@@ -1029,20 +1038,17 @@ mod tests {
10291038
}
10301039

10311040
#[test]
1032-
fn test_distinct_count_partial_ndv_in_row_groups() {
1033-
// Some row groups have NDV, some don't - should use only those that have it
1041+
fn test_distinct_count_partial_ndv_below_threshold() {
1042+
// 1 of 2 row groups has NDV (50% < 75% threshold) -> Absent
10341043
let schema_descr = create_schema_descr(1);
10351044
let arrow_schema = create_arrow_schema(1);
10361045

1037-
// Row group 1: has distinct_count = 15
10381046
let stats1 =
10391047
ParquetStatistics::int32(Some(1), Some(50), Some(15), Some(0), false);
1040-
1041-
// Row group 2: no distinct_count
10421048
let stats2 = ParquetStatistics::int32(
10431049
Some(51),
10441050
Some(100),
1045-
None, // no distinct_count
1051+
None,
10461052
Some(0),
10471053
false,
10481054
);
@@ -1060,10 +1066,39 @@ mod tests {
10601066
)
10611067
.unwrap();
10621068

1063-
// Only one row group has NDV, so it's Exact(15)
10641069
assert_eq!(
10651070
result.column_statistics[0].distinct_count,
1066-
Precision::Exact(15)
1071+
Precision::Absent
1072+
);
1073+
}
1074+
1075+
#[test]
1076+
fn test_distinct_count_partial_ndv_above_threshold() {
1077+
// 3 of 4 row groups have NDV (75% >= 75% threshold) -> Inexact
1078+
let schema_descr = create_schema_descr(1);
1079+
let arrow_schema = create_arrow_schema(1);
1080+
1081+
let stats_with =
1082+
|ndv| ParquetStatistics::int32(Some(1), Some(100), Some(ndv), Some(0), false);
1083+
let stats_without =
1084+
ParquetStatistics::int32(Some(1), Some(100), None, Some(0), false);
1085+
1086+
let rg1 = create_row_group_with_stats(&schema_descr, vec![Some(stats_with(10))], 250);
1087+
let rg2 = create_row_group_with_stats(&schema_descr, vec![Some(stats_with(20))], 250);
1088+
let rg3 = create_row_group_with_stats(&schema_descr, vec![Some(stats_with(15))], 250);
1089+
let rg4 = create_row_group_with_stats(&schema_descr, vec![Some(stats_without)], 250);
1090+
let metadata =
1091+
create_parquet_metadata(schema_descr, vec![rg1, rg2, rg3, rg4]);
1092+
1093+
let result = DFParquetMetadata::statistics_from_parquet_metadata(
1094+
&metadata,
1095+
&arrow_schema,
1096+
)
1097+
.unwrap();
1098+
1099+
assert_eq!(
1100+
result.column_statistics[0].distinct_count,
1101+
Precision::Inexact(20)
10671102
);
10681103
}
10691104

0 commit comments

Comments
 (0)