@@ -204,17 +204,6 @@ pub struct FileScanConfig {
204204 /// would be incorrect if there are filters being applied, thus this should be accessed
205205 /// via [`FileScanConfig::statistics`].
206206 pub ( crate ) statistics : Statistics ,
207- /// When true, file_groups are organized by partition column values
208- /// and output_partitioning will return Hash partitioning on partition columns.
209- /// This allows the optimizer to skip hash repartitioning for aggregates and joins
210- /// on partition columns.
211- ///
212- /// If the number of file partitions > target_partitions, the file partitions will be grouped
213- /// in a round-robin fashion such that number of file partitions = target_partitions.
214- ///
215- /// Follow-up: remove this redundant field in favor of
216- /// `output_partitioning`, see <https://github.com/apache/datafusion/issues/23099>.
217- pub partitioned_by_file_group : bool ,
218207 /// Declared physical output partitioning for this scan.
219208 ///
220209 /// Expressions are against the full table schema, before scan projection or
@@ -294,7 +283,6 @@ pub struct FileScanConfigBuilder {
294283 file_compression_type : Option < FileCompressionType > ,
295284 batch_size : Option < usize > ,
296285 expr_adapter_factory : Option < Arc < dyn PhysicalExprAdapterFactory > > ,
297- partitioned_by_file_group : bool ,
298286}
299287
300288impl FileScanConfigBuilder {
@@ -321,7 +309,6 @@ impl FileScanConfigBuilder {
321309 constraints : None ,
322310 batch_size : None ,
323311 expr_adapter_factory : None ,
324- partitioned_by_file_group : false ,
325312 }
326313 }
327314
@@ -519,18 +506,6 @@ impl FileScanConfigBuilder {
519506 self
520507 }
521508
522- /// Set whether file groups are organized by partition column values.
523- ///
524- /// When set to true, the output partitioning will be declared as Hash partitioning
525- /// on the partition columns.
526- pub fn with_partitioned_by_file_group (
527- mut self ,
528- partitioned_by_file_group : bool ,
529- ) -> Self {
530- self . partitioned_by_file_group = partitioned_by_file_group;
531- self
532- }
533-
534509 /// Build the final [`FileScanConfig`] with all the configured settings.
535510 ///
536511 /// This method takes ownership of the builder and returns the constructed `FileScanConfig`.
@@ -552,7 +527,6 @@ impl FileScanConfigBuilder {
552527 file_compression_type,
553528 batch_size,
554529 expr_adapter_factory : expr_adapter,
555- partitioned_by_file_group,
556530 } = self ;
557531
558532 let constraints = constraints. unwrap_or_default ( ) ;
@@ -577,7 +551,6 @@ impl FileScanConfigBuilder {
577551 batch_size,
578552 expr_adapter_factory : expr_adapter,
579553 statistics,
580- partitioned_by_file_group,
581554 output_partitioning,
582555 }
583556 }
@@ -598,12 +571,15 @@ impl From<FileScanConfig> for FileScanConfigBuilder {
598571 constraints : Some ( config. constraints ) ,
599572 batch_size : config. batch_size ,
600573 expr_adapter_factory : config. expr_adapter_factory ,
601- partitioned_by_file_group : config. partitioned_by_file_group ,
602574 }
603575 }
604576}
605577
606- fn hash_partitioning_from_partition_fields (
578+ /// Builds `Partitioning::Hash` over `partition_cols` (resolved to their indices in
579+ /// `schema`) with `partition_count` partitions. Returns `None` when there are no
580+ /// partition columns. Callers use this to declare the output partitioning of a scan
581+ /// whose file groups are organized by partition column values.
582+ pub fn hash_partitioning_from_partition_fields (
607583 schema : & Schema ,
608584 partition_cols : & Fields ,
609585 partition_count : usize ,
@@ -765,7 +741,7 @@ impl DataSource for FileScanConfig {
765741 ) -> Result < Option < Arc < dyn DataSource > > > {
766742 // When file groups define output partitioning, repartitioning files
767743 // would invalidate the partition-to-file-group mapping.
768- if self . output_partitioning . is_some ( ) || self . partitioned_by_file_group {
744+ if self . output_partitioning . is_some ( ) {
769745 return Ok ( None ) ;
770746 }
771747
@@ -782,10 +758,8 @@ impl DataSource for FileScanConfig {
782758 /// Returns the output partitioning for this file scan.
783759 ///
784760 /// When `output_partitioning` is set, this returns the declared partitioning
785- /// after applying scan projection. When `partitioned_by_file_group` is true,
786- /// this returns `Partitioning::Hash` on the Hive partition columns, allowing
787- /// the optimizer to skip hash repartitioning for aggregates and joins on
788- /// those columns.
761+ /// after applying scan projection, allowing the optimizer to skip hash
762+ /// repartitioning for aggregates and joins on the partitioning columns.
789763 ///
790764 /// If projection or partition count validation fails, this returns
791765 /// `UnknownPartitioning`.
@@ -801,15 +775,7 @@ impl DataSource for FileScanConfig {
801775 /// - Idea: Could allow byte-range splitting within partition-aware groups,
802776 /// preserving I/O parallelism while maintaining partition semantics.
803777 fn output_partitioning ( & self ) -> Partitioning {
804- let Some ( output_partitioning) = self . output_partitioning . clone ( ) . or_else ( || {
805- self . partitioned_by_file_group . then ( || {
806- hash_partitioning_from_partition_fields (
807- self . file_source . table_schema ( ) . table_schema ( ) ,
808- self . table_partition_cols ( ) ,
809- self . file_groups . len ( ) ,
810- )
811- } ) ?
812- } ) else {
778+ let Some ( output_partitioning) = self . output_partitioning . clone ( ) else {
813779 return Partitioning :: UnknownPartitioning ( self . file_groups . len ( ) ) ;
814780 } ;
815781 if output_partitioning. partition_count ( ) != self . file_groups . len ( ) {
@@ -1140,7 +1106,6 @@ impl DataSource for FileScanConfig {
11401106 ) -> Option < Arc < dyn Any + Send + Sync > > {
11411107 if self . preserve_order
11421108 || self . output_partitioning . is_some ( )
1143- || self . partitioned_by_file_group
11441109 || !config. execution . enable_file_stream_work_stealing
11451110 {
11461111 return None ;
@@ -2553,7 +2518,7 @@ mod tests {
25532518 vec ! [ partition_col] ,
25542519 ) ;
25552520
2556- // partitioned_by_file_group defaults to false
2521+ // output_partitioning defaults to None
25572522 let partitioning = config. output_partitioning ( ) ;
25582523 assert ! ( matches!( partitioning, Partitioning :: UnknownPartitioning ( _) ) ) ;
25592524 }
@@ -2613,13 +2578,12 @@ mod tests {
26132578 #[ test]
26142579 fn test_output_partitioning_no_partition_columns ( ) {
26152580 let file_schema = aggr_test_schema ( ) ;
2616- let mut config = config_for_projection (
2581+ let config = config_for_projection (
26172582 Arc :: clone ( & file_schema) ,
26182583 None ,
26192584 Statistics :: new_unknown ( & file_schema) ,
26202585 vec ! [ ] , // No partition columns
26212586 ) ;
2622- config. partitioned_by_file_group = true ;
26232587
26242588 let partitioning = config. output_partitioning ( ) ;
26252589 assert ! ( matches!( partitioning, Partitioning :: UnknownPartitioning ( _) ) ) ;
@@ -2642,12 +2606,16 @@ mod tests {
26422606 Statistics :: new_unknown ( & file_schema) ,
26432607 single_partition_col,
26442608 ) ;
2645- config. partitioned_by_file_group = true ;
26462609 config. file_groups = vec ! [
26472610 FileGroup :: new( vec![ PartitionedFile :: new( "f1.parquet" . to_string( ) , 1024 ) ] ) ,
26482611 FileGroup :: new( vec![ PartitionedFile :: new( "f2.parquet" . to_string( ) , 1024 ) ] ) ,
26492612 FileGroup :: new( vec![ PartitionedFile :: new( "f3.parquet" . to_string( ) , 1024 ) ] ) ,
26502613 ] ;
2614+ config. output_partitioning = hash_partitioning_from_partition_fields (
2615+ config. file_source . table_schema ( ) . table_schema ( ) ,
2616+ config. table_partition_cols ( ) ,
2617+ config. file_groups . len ( ) ,
2618+ ) ;
26512619
26522620 let partitioning = config. output_partitioning ( ) ;
26532621 match partitioning {
@@ -2671,11 +2639,15 @@ mod tests {
26712639 Statistics :: new_unknown ( & file_schema) ,
26722640 multiple_partition_cols,
26732641 ) ;
2674- config. partitioned_by_file_group = true ;
26752642 config. file_groups = vec ! [
26762643 FileGroup :: new( vec![ PartitionedFile :: new( "f1.parquet" . to_string( ) , 1024 ) ] ) ,
26772644 FileGroup :: new( vec![ PartitionedFile :: new( "f2.parquet" . to_string( ) , 1024 ) ] ) ,
26782645 ] ;
2646+ config. output_partitioning = hash_partitioning_from_partition_fields (
2647+ config. file_source . table_schema ( ) . table_schema ( ) ,
2648+ config. table_partition_cols ( ) ,
2649+ config. file_groups . len ( ) ,
2650+ ) ;
26792651
26802652 let partitioning = config. output_partitioning ( ) ;
26812653 match partitioning {
0 commit comments