@@ -198,17 +198,6 @@ pub struct FileScanConfig {
198198 /// would be incorrect if there are filters being applied, thus this should be accessed
199199 /// via [`FileScanConfig::statistics`].
200200 pub ( crate ) statistics : Statistics ,
201- /// When true, file_groups are organized by partition column values
202- /// and output_partitioning will return Hash partitioning on partition columns.
203- /// This allows the optimizer to skip hash repartitioning for aggregates and joins
204- /// on partition columns.
205- ///
206- /// If the number of file partitions > target_partitions, the file partitions will be grouped
207- /// in a round-robin fashion such that number of file partitions = target_partitions.
208- ///
209- /// Follow-up: remove this redundant field in favor of
210- /// `output_partitioning`, see <https://github.com/apache/datafusion/issues/23099>.
211- pub partitioned_by_file_group : bool ,
212201 /// Declared physical output partitioning for this scan.
213202 ///
214203 /// Expressions are against the full table schema, before scan projection or
@@ -288,7 +277,6 @@ pub struct FileScanConfigBuilder {
288277 file_compression_type : Option < FileCompressionType > ,
289278 batch_size : Option < usize > ,
290279 expr_adapter_factory : Option < Arc < dyn PhysicalExprAdapterFactory > > ,
291- partitioned_by_file_group : bool ,
292280}
293281
294282impl FileScanConfigBuilder {
@@ -315,7 +303,6 @@ impl FileScanConfigBuilder {
315303 constraints : None ,
316304 batch_size : None ,
317305 expr_adapter_factory : None ,
318- partitioned_by_file_group : false ,
319306 }
320307 }
321308
@@ -513,18 +500,6 @@ impl FileScanConfigBuilder {
513500 self
514501 }
515502
516- /// Set whether file groups are organized by partition column values.
517- ///
518- /// When set to true, the output partitioning will be declared as Hash partitioning
519- /// on the partition columns.
520- pub fn with_partitioned_by_file_group (
521- mut self ,
522- partitioned_by_file_group : bool ,
523- ) -> Self {
524- self . partitioned_by_file_group = partitioned_by_file_group;
525- self
526- }
527-
528503 /// Build the final [`FileScanConfig`] with all the configured settings.
529504 ///
530505 /// This method takes ownership of the builder and returns the constructed `FileScanConfig`.
@@ -546,7 +521,6 @@ impl FileScanConfigBuilder {
546521 file_compression_type,
547522 batch_size,
548523 expr_adapter_factory : expr_adapter,
549- partitioned_by_file_group,
550524 } = self ;
551525
552526 let constraints = constraints. unwrap_or_default ( ) ;
@@ -571,7 +545,6 @@ impl FileScanConfigBuilder {
571545 batch_size,
572546 expr_adapter_factory : expr_adapter,
573547 statistics,
574- partitioned_by_file_group,
575548 output_partitioning,
576549 }
577550 }
@@ -592,12 +565,15 @@ impl From<FileScanConfig> for FileScanConfigBuilder {
592565 constraints : Some ( config. constraints ) ,
593566 batch_size : config. batch_size ,
594567 expr_adapter_factory : config. expr_adapter_factory ,
595- partitioned_by_file_group : config. partitioned_by_file_group ,
596568 }
597569 }
598570}
599571
600- fn hash_partitioning_from_partition_fields (
572+ /// Builds `Partitioning::Hash` over `partition_cols` (resolved to their indices in
573+ /// `schema`) with `partition_count` partitions. Returns `None` when there are no
574+ /// partition columns. Callers use this to declare the output partitioning of a scan
575+ /// whose file groups are organized by partition column values.
576+ pub fn hash_partitioning_from_partition_fields (
601577 schema : & Schema ,
602578 partition_cols : & Fields ,
603579 partition_count : usize ,
@@ -759,7 +735,7 @@ impl DataSource for FileScanConfig {
759735 ) -> Result < Option < Arc < dyn DataSource > > > {
760736 // When file groups define output partitioning, repartitioning files
761737 // would invalidate the partition-to-file-group mapping.
762- if self . output_partitioning . is_some ( ) || self . partitioned_by_file_group {
738+ if self . output_partitioning . is_some ( ) {
763739 return Ok ( None ) ;
764740 }
765741
@@ -776,10 +752,8 @@ impl DataSource for FileScanConfig {
776752 /// Returns the output partitioning for this file scan.
777753 ///
778754 /// When `output_partitioning` is set, this returns the declared partitioning
779- /// after applying scan projection. When `partitioned_by_file_group` is true,
780- /// this returns `Partitioning::Hash` on the Hive partition columns, allowing
781- /// the optimizer to skip hash repartitioning for aggregates and joins on
782- /// those columns.
755+ /// after applying scan projection, allowing the optimizer to skip hash
756+ /// repartitioning for aggregates and joins on the partitioning columns.
783757 ///
784758 /// If projection or partition count validation fails, this returns
785759 /// `UnknownPartitioning`.
@@ -795,15 +769,7 @@ impl DataSource for FileScanConfig {
795769 /// - Idea: Could allow byte-range splitting within partition-aware groups,
796770 /// preserving I/O parallelism while maintaining partition semantics.
797771 fn output_partitioning ( & self ) -> Partitioning {
798- let Some ( output_partitioning) = self . output_partitioning . clone ( ) . or_else ( || {
799- self . partitioned_by_file_group . then ( || {
800- hash_partitioning_from_partition_fields (
801- self . file_source . table_schema ( ) . table_schema ( ) ,
802- self . table_partition_cols ( ) ,
803- self . file_groups . len ( ) ,
804- )
805- } ) ?
806- } ) else {
772+ let Some ( output_partitioning) = self . output_partitioning . clone ( ) else {
807773 return Partitioning :: UnknownPartitioning ( self . file_groups . len ( ) ) ;
808774 } ;
809775 if output_partitioning. partition_count ( ) != self . file_groups . len ( ) {
@@ -1127,10 +1093,7 @@ impl DataSource for FileScanConfig {
11271093 /// when file order must be preserved or the file groups define the output
11281094 /// partitioning needed for the rest of the plan
11291095 fn create_sibling_state ( & self ) -> Option < Arc < dyn Any + Send + Sync > > {
1130- if self . preserve_order
1131- || self . output_partitioning . is_some ( )
1132- || self . partitioned_by_file_group
1133- {
1096+ if self . preserve_order || self . output_partitioning . is_some ( ) {
11341097 return None ;
11351098 }
11361099
@@ -2541,7 +2504,7 @@ mod tests {
25412504 vec ! [ partition_col] ,
25422505 ) ;
25432506
2544- // partitioned_by_file_group defaults to false
2507+ // output_partitioning defaults to None
25452508 let partitioning = config. output_partitioning ( ) ;
25462509 assert ! ( matches!( partitioning, Partitioning :: UnknownPartitioning ( _) ) ) ;
25472510 }
@@ -2601,13 +2564,12 @@ mod tests {
26012564 #[ test]
26022565 fn test_output_partitioning_no_partition_columns ( ) {
26032566 let file_schema = aggr_test_schema ( ) ;
2604- let mut config = config_for_projection (
2567+ let config = config_for_projection (
26052568 Arc :: clone ( & file_schema) ,
26062569 None ,
26072570 Statistics :: new_unknown ( & file_schema) ,
26082571 vec ! [ ] , // No partition columns
26092572 ) ;
2610- config. partitioned_by_file_group = true ;
26112573
26122574 let partitioning = config. output_partitioning ( ) ;
26132575 assert ! ( matches!( partitioning, Partitioning :: UnknownPartitioning ( _) ) ) ;
@@ -2630,12 +2592,16 @@ mod tests {
26302592 Statistics :: new_unknown ( & file_schema) ,
26312593 single_partition_col,
26322594 ) ;
2633- config. partitioned_by_file_group = true ;
26342595 config. file_groups = vec ! [
26352596 FileGroup :: new( vec![ PartitionedFile :: new( "f1.parquet" . to_string( ) , 1024 ) ] ) ,
26362597 FileGroup :: new( vec![ PartitionedFile :: new( "f2.parquet" . to_string( ) , 1024 ) ] ) ,
26372598 FileGroup :: new( vec![ PartitionedFile :: new( "f3.parquet" . to_string( ) , 1024 ) ] ) ,
26382599 ] ;
2600+ config. output_partitioning = hash_partitioning_from_partition_fields (
2601+ config. file_source . table_schema ( ) . table_schema ( ) ,
2602+ config. table_partition_cols ( ) ,
2603+ config. file_groups . len ( ) ,
2604+ ) ;
26392605
26402606 let partitioning = config. output_partitioning ( ) ;
26412607 match partitioning {
@@ -2659,11 +2625,15 @@ mod tests {
26592625 Statistics :: new_unknown ( & file_schema) ,
26602626 multiple_partition_cols,
26612627 ) ;
2662- config. partitioned_by_file_group = true ;
26632628 config. file_groups = vec ! [
26642629 FileGroup :: new( vec![ PartitionedFile :: new( "f1.parquet" . to_string( ) , 1024 ) ] ) ,
26652630 FileGroup :: new( vec![ PartitionedFile :: new( "f2.parquet" . to_string( ) , 1024 ) ] ) ,
26662631 ] ;
2632+ config. output_partitioning = hash_partitioning_from_partition_fields (
2633+ config. file_source . table_schema ( ) . table_schema ( ) ,
2634+ config. table_partition_cols ( ) ,
2635+ config. file_groups . len ( ) ,
2636+ ) ;
26672637
26682638 let partitioning = config. output_partitioning ( ) ;
26692639 match partitioning {
0 commit comments