Skip to content

Commit 32d17cb

Browse files
Dandandanclaude
andcommitted
feat: derive a distinct count from primary key and unique constraints
A join has to estimate how many rows it produces, which needs the number of distinct values in each key. No file format stores one, so a scan reports it as unknown even for a column the table declares unique, and the estimate is a guess. A single-column primary key or unique constraint gives that number directly: one distinct value per row, less the nulls a unique column may repeat. A composite key says nothing about its columns on their own, so it is left alone, and the count is inexact because constraints are not verified. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_012YiAABcW4WSqij31zz2P6c
1 parent 858d3bf commit 32d17cb

1 file changed

Lines changed: 131 additions & 2 deletions

File tree

  • datafusion/datasource/src/file_scan_config

‎datafusion/datasource/src/file_scan_config/mod.rs‎

Lines changed: 131 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,8 @@ use arrow::datatypes::{DataType, Schema, SchemaRef};
3939
use datafusion_common::config::ConfigOptions;
4040
use datafusion_common::tree_node::TreeNodeRecursion;
4141
use datafusion_common::{
42-
Constraints, Result, ScalarValue, Statistics, internal_datafusion_err, internal_err,
42+
Constraint, Constraints, Result, ScalarValue, Statistics, internal_datafusion_err,
43+
internal_err,
4344
};
4445
use datafusion_execution::{
4546
SendableRecordBatchStream, TaskContext, object_store::ObjectStoreUrl,
@@ -1267,10 +1268,43 @@ impl FileScanConfig {
12671268
pub fn statistics(&self) -> Statistics {
12681269
let filter_may_change_row_count = self.file_source.filter().is_some()
12691270
&& self.statistics.num_rows != Precision::Exact(0);
1270-
if filter_may_change_row_count {
1271+
let mut statistics = if filter_may_change_row_count {
12711272
self.statistics.clone().to_inexact()
12721273
} else {
12731274
self.statistics.clone()
1275+
};
1276+
self.add_key_distinct_counts(&mut statistics);
1277+
statistics
1278+
}
1279+
1280+
/// Records that a key column holds one distinct value per row.
1281+
///
1282+
/// No file format stores a distinct count, so without this a join on a key has to
1283+
/// guess how many rows it produces. Constraints are not verified, hence inexact.
1284+
/// A composite key says nothing about its columns on their own, so only
1285+
/// single-column keys are used.
1286+
fn add_key_distinct_counts(&self, statistics: &mut Statistics) {
1287+
let num_rows = statistics.num_rows.to_inexact();
1288+
for constraint in self.constraints.iter() {
1289+
let (Constraint::PrimaryKey(indices) | Constraint::Unique(indices)) =
1290+
constraint;
1291+
let [index] = indices[..] else {
1292+
continue;
1293+
};
1294+
let Some(column) = statistics.column_statistics.get_mut(index) else {
1295+
continue;
1296+
};
1297+
if column.distinct_count != Precision::Absent {
1298+
continue;
1299+
}
1300+
// A unique column may hold NULL more than once, and a NULL is not a
1301+
// distinct value. A primary key cannot be null, and an unknown null count
1302+
// is taken as none.
1303+
let nulls = match column.null_count {
1304+
Precision::Absent => Precision::Inexact(0),
1305+
nulls => nulls,
1306+
};
1307+
column.distinct_count = num_rows.sub(&nulls);
12741308
}
12751309
}
12761310

@@ -2133,6 +2167,101 @@ mod tests {
21332167
}
21342168

21352169
// sets default for configs that play no role in projections
2170+
fn config_with_constraints(
2171+
table_schema: TableSchema,
2172+
statistics: Statistics,
2173+
constraints: Vec<Constraint>,
2174+
projection: Option<Vec<usize>>,
2175+
) -> FileScanConfig {
2176+
FileScanConfigBuilder::new(
2177+
ObjectStoreUrl::parse("test:///").unwrap(),
2178+
Arc::new(MockSource::new(table_schema)),
2179+
)
2180+
.with_statistics(statistics)
2181+
.with_constraints(Constraints::new_unverified(constraints))
2182+
.with_projection_indices(projection)
2183+
.unwrap()
2184+
.build()
2185+
}
2186+
2187+
/// A key column has one distinct value per row, which no file format records.
2188+
#[test]
2189+
fn key_columns_report_a_distinct_count() {
2190+
let file_schema = Arc::new(Schema::new(vec![
2191+
Field::new("id", DataType::Int32, false),
2192+
Field::new("part", DataType::Int32, false),
2193+
Field::new("code", DataType::Int32, true),
2194+
]));
2195+
let table_schema = TableSchema::builder(Arc::clone(&file_schema)).build();
2196+
let mut statistics = Statistics::new_unknown(&file_schema);
2197+
statistics.num_rows = Precision::Exact(100);
2198+
// A unique column may repeat NULL, which is not a distinct value.
2199+
statistics.column_statistics[2].null_count = Precision::Exact(10);
2200+
2201+
let stats = |constraints| {
2202+
config_with_constraints(
2203+
table_schema.clone(),
2204+
statistics.clone(),
2205+
constraints,
2206+
None,
2207+
)
2208+
.statistics()
2209+
};
2210+
2211+
// Unverified constraints, so the count is inexact.
2212+
let primary_key = stats(vec![Constraint::PrimaryKey(vec![0])]);
2213+
assert_eq!(
2214+
primary_key.column_statistics[0].distinct_count,
2215+
Precision::Inexact(100)
2216+
);
2217+
assert_eq!(
2218+
primary_key.column_statistics[1].distinct_count,
2219+
Precision::Absent
2220+
);
2221+
2222+
let unique = stats(vec![Constraint::Unique(vec![2])]);
2223+
assert_eq!(
2224+
unique.column_statistics[2].distinct_count,
2225+
Precision::Inexact(90)
2226+
);
2227+
2228+
// A composite key leaves its columns alone: only the combination is unique.
2229+
let composite = stats(vec![Constraint::PrimaryKey(vec![0, 1])]);
2230+
assert_eq!(
2231+
composite.column_statistics[0].distinct_count,
2232+
Precision::Absent
2233+
);
2234+
assert_eq!(
2235+
composite.column_statistics[1].distinct_count,
2236+
Precision::Absent
2237+
);
2238+
}
2239+
2240+
/// The count has to reach the plan, which reads statistics through the projection.
2241+
#[test]
2242+
fn a_projected_scan_keeps_the_key_distinct_count() {
2243+
let file_schema = Arc::new(Schema::new(vec![
2244+
Field::new("value", DataType::Int32, true),
2245+
Field::new("id", DataType::Int32, false),
2246+
]));
2247+
let table_schema = TableSchema::builder(Arc::clone(&file_schema)).build();
2248+
let mut statistics = Statistics::new_unknown(&file_schema);
2249+
statistics.num_rows = Precision::Exact(100);
2250+
2251+
let config = config_with_constraints(
2252+
table_schema,
2253+
statistics,
2254+
vec![Constraint::PrimaryKey(vec![1])],
2255+
Some(vec![1]),
2256+
);
2257+
2258+
let projected = config.partition_statistics(None).unwrap();
2259+
assert_eq!(
2260+
projected.column_statistics[0].distinct_count,
2261+
Precision::Inexact(100)
2262+
);
2263+
}
2264+
21362265
fn config_for_projection(
21372266
file_schema: SchemaRef,
21382267
projection: Option<Vec<usize>>,

0 commit comments

Comments
 (0)