@@ -39,7 +39,8 @@ use arrow::datatypes::{DataType, Schema, SchemaRef};
3939use datafusion_common:: config:: ConfigOptions ;
4040use datafusion_common:: tree_node:: TreeNodeRecursion ;
4141use datafusion_common:: {
42- Constraints , Result , ScalarValue , Statistics , internal_datafusion_err, internal_err,
42+ Constraint , Constraints , Result , ScalarValue , Statistics , internal_datafusion_err,
43+ internal_err,
4344} ;
4445use datafusion_execution:: {
4546 SendableRecordBatchStream , TaskContext , object_store:: ObjectStoreUrl ,
@@ -1267,10 +1268,43 @@ impl FileScanConfig {
12671268 pub fn statistics ( & self ) -> Statistics {
12681269 let filter_may_change_row_count = self . file_source . filter ( ) . is_some ( )
12691270 && self . statistics . num_rows != Precision :: Exact ( 0 ) ;
1270- if filter_may_change_row_count {
1271+ let mut statistics = if filter_may_change_row_count {
12711272 self . statistics . clone ( ) . to_inexact ( )
12721273 } else {
12731274 self . statistics . clone ( )
1275+ } ;
1276+ self . add_key_distinct_counts ( & mut statistics) ;
1277+ statistics
1278+ }
1279+
1280+ /// Records that a key column holds one distinct value per row.
1281+ ///
1282+ /// No file format stores a distinct count, so without this a join on a key has to
1283+ /// guess how many rows it produces. Constraints are not verified, hence inexact.
1284+ /// A composite key says nothing about its columns on their own, so only
1285+ /// single-column keys are used.
1286+ fn add_key_distinct_counts ( & self , statistics : & mut Statistics ) {
1287+ let num_rows = statistics. num_rows . to_inexact ( ) ;
1288+ for constraint in self . constraints . iter ( ) {
1289+ let ( Constraint :: PrimaryKey ( indices) | Constraint :: Unique ( indices) ) =
1290+ constraint;
1291+ let [ index] = indices[ ..] else {
1292+ continue ;
1293+ } ;
1294+ let Some ( column) = statistics. column_statistics . get_mut ( index) else {
1295+ continue ;
1296+ } ;
1297+ if column. distinct_count != Precision :: Absent {
1298+ continue ;
1299+ }
1300+ // A unique column may hold NULL more than once, and a NULL is not a
1301+ // distinct value. A primary key cannot be null, and an unknown null count
1302+ // is taken as none.
1303+ let nulls = match column. null_count {
1304+ Precision :: Absent => Precision :: Inexact ( 0 ) ,
1305+ nulls => nulls,
1306+ } ;
1307+ column. distinct_count = num_rows. sub ( & nulls) ;
12741308 }
12751309 }
12761310
@@ -2133,6 +2167,101 @@ mod tests {
21332167 }
21342168
21352169 // sets default for configs that play no role in projections
2170+ fn config_with_constraints (
2171+ table_schema : TableSchema ,
2172+ statistics : Statistics ,
2173+ constraints : Vec < Constraint > ,
2174+ projection : Option < Vec < usize > > ,
2175+ ) -> FileScanConfig {
2176+ FileScanConfigBuilder :: new (
2177+ ObjectStoreUrl :: parse ( "test:///" ) . unwrap ( ) ,
2178+ Arc :: new ( MockSource :: new ( table_schema) ) ,
2179+ )
2180+ . with_statistics ( statistics)
2181+ . with_constraints ( Constraints :: new_unverified ( constraints) )
2182+ . with_projection_indices ( projection)
2183+ . unwrap ( )
2184+ . build ( )
2185+ }
2186+
2187+ /// A key column has one distinct value per row, which no file format records.
2188+ #[ test]
2189+ fn key_columns_report_a_distinct_count ( ) {
2190+ let file_schema = Arc :: new ( Schema :: new ( vec ! [
2191+ Field :: new( "id" , DataType :: Int32 , false ) ,
2192+ Field :: new( "part" , DataType :: Int32 , false ) ,
2193+ Field :: new( "code" , DataType :: Int32 , true ) ,
2194+ ] ) ) ;
2195+ let table_schema = TableSchema :: builder ( Arc :: clone ( & file_schema) ) . build ( ) ;
2196+ let mut statistics = Statistics :: new_unknown ( & file_schema) ;
2197+ statistics. num_rows = Precision :: Exact ( 100 ) ;
2198+ // A unique column may repeat NULL, which is not a distinct value.
2199+ statistics. column_statistics [ 2 ] . null_count = Precision :: Exact ( 10 ) ;
2200+
2201+ let stats = |constraints| {
2202+ config_with_constraints (
2203+ table_schema. clone ( ) ,
2204+ statistics. clone ( ) ,
2205+ constraints,
2206+ None ,
2207+ )
2208+ . statistics ( )
2209+ } ;
2210+
2211+ // Unverified constraints, so the count is inexact.
2212+ let primary_key = stats ( vec ! [ Constraint :: PrimaryKey ( vec![ 0 ] ) ] ) ;
2213+ assert_eq ! (
2214+ primary_key. column_statistics[ 0 ] . distinct_count,
2215+ Precision :: Inexact ( 100 )
2216+ ) ;
2217+ assert_eq ! (
2218+ primary_key. column_statistics[ 1 ] . distinct_count,
2219+ Precision :: Absent
2220+ ) ;
2221+
2222+ let unique = stats ( vec ! [ Constraint :: Unique ( vec![ 2 ] ) ] ) ;
2223+ assert_eq ! (
2224+ unique. column_statistics[ 2 ] . distinct_count,
2225+ Precision :: Inexact ( 90 )
2226+ ) ;
2227+
2228+ // A composite key leaves its columns alone: only the combination is unique.
2229+ let composite = stats ( vec ! [ Constraint :: PrimaryKey ( vec![ 0 , 1 ] ) ] ) ;
2230+ assert_eq ! (
2231+ composite. column_statistics[ 0 ] . distinct_count,
2232+ Precision :: Absent
2233+ ) ;
2234+ assert_eq ! (
2235+ composite. column_statistics[ 1 ] . distinct_count,
2236+ Precision :: Absent
2237+ ) ;
2238+ }
2239+
2240+ /// The count has to reach the plan, which reads statistics through the projection.
2241+ #[ test]
2242+ fn a_projected_scan_keeps_the_key_distinct_count ( ) {
2243+ let file_schema = Arc :: new ( Schema :: new ( vec ! [
2244+ Field :: new( "value" , DataType :: Int32 , true ) ,
2245+ Field :: new( "id" , DataType :: Int32 , false ) ,
2246+ ] ) ) ;
2247+ let table_schema = TableSchema :: builder ( Arc :: clone ( & file_schema) ) . build ( ) ;
2248+ let mut statistics = Statistics :: new_unknown ( & file_schema) ;
2249+ statistics. num_rows = Precision :: Exact ( 100 ) ;
2250+
2251+ let config = config_with_constraints (
2252+ table_schema,
2253+ statistics,
2254+ vec ! [ Constraint :: PrimaryKey ( vec![ 1 ] ) ] ,
2255+ Some ( vec ! [ 1 ] ) ,
2256+ ) ;
2257+
2258+ let projected = config. partition_statistics ( None ) . unwrap ( ) ;
2259+ assert_eq ! (
2260+ projected. column_statistics[ 0 ] . distinct_count,
2261+ Precision :: Inexact ( 100 )
2262+ ) ;
2263+ }
2264+
21362265 fn config_for_projection (
21372266 file_schema : SchemaRef ,
21382267 projection : Option < Vec < usize > > ,
0 commit comments