Skip to content

Commit ca000a2

Browse files
committed
refactor: remove redundant partitioned_by_file_group file scan field
`FileScanConfig` had two overlapping ways to declare file scan output partitioning: the `partitioned_by_file_group` bool and `output_partitioning`. Collapse them onto `output_partitioning` as the single source of truth. - Remove the `partitioned_by_file_group` field, the builder field, and the `with_partitioned_by_file_group` builder method. - `ListingTable` now derives the partition-column `Partitioning::Hash` once its file groups are finalized and passes it via `with_output_partitioning`; `hash_partitioning_from_partition_fields` is made `pub` for this. - proto already round-trips `output_partitioning`, so the now-vestigial wire bool is left unset on write and ignored on read (the proto field is kept for backward compatibility). Closes apache#23099. Signed-off-by: Jiawei Zhao <[email protected]>
1 parent 094ad31 commit ca000a2

6 files changed

Lines changed: 50 additions & 77 deletions

File tree

‎datafusion/catalog-listing/src/table.rs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@ use datafusion_common::{
3030
};
3131
use datafusion_datasource::file::FileSource;
3232
use datafusion_datasource::file_groups::FileGroup;
33-
use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};
33+
use datafusion_datasource::file_scan_config::{
34+
FileScanConfig, FileScanConfigBuilder, hash_partitioning_from_partition_fields,
35+
};
3436
use datafusion_datasource::file_sink_config::{FileOutputMode, FileSinkConfig};
3537
#[expect(deprecated)]
3638
use datafusion_datasource::schema_adapter::SchemaAdapterFactory;
@@ -628,6 +630,15 @@ impl TableProvider for ListingTable {
628630
);
629631
}
630632
Some(output_partitioning)
633+
} else if partitioned_by_file_group {
634+
// Files are grouped by partition column values: declare Hash
635+
// partitioning on those columns so the optimizer can skip hash
636+
// repartitioning for aggregates and joins on the partition columns.
637+
hash_partitioning_from_partition_fields(
638+
&self.table_schema,
639+
&table_partition_cols.clone().into(),
640+
partitioned_file_lists.len(),
641+
)
631642
} else {
632643
None
633644
};
@@ -650,7 +661,6 @@ impl TableProvider for ListingTable {
650661
.with_output_ordering(output_ordering)
651662
.with_output_partitioning(output_partitioning)
652663
.with_expr_adapter(self.expr_adapter_factory.clone())
653-
.with_partitioned_by_file_group(partitioned_by_file_group)
654664
.build();
655665

656666
// create the execution plan

‎datafusion/datasource/src/file_scan_config/mod.rs‎

Lines changed: 21 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -204,17 +204,6 @@ pub struct FileScanConfig {
204204
/// would be incorrect if there are filters being applied, thus this should be accessed
205205
/// via [`FileScanConfig::statistics`].
206206
pub(crate) statistics: Statistics,
207-
/// When true, file_groups are organized by partition column values
208-
/// and output_partitioning will return Hash partitioning on partition columns.
209-
/// This allows the optimizer to skip hash repartitioning for aggregates and joins
210-
/// on partition columns.
211-
///
212-
/// If the number of file partitions > target_partitions, the file partitions will be grouped
213-
/// in a round-robin fashion such that number of file partitions = target_partitions.
214-
///
215-
/// Follow-up: remove this redundant field in favor of
216-
/// `output_partitioning`, see <https://github.com/apache/datafusion/issues/23099>.
217-
pub partitioned_by_file_group: bool,
218207
/// Declared physical output partitioning for this scan.
219208
///
220209
/// Expressions are against the full table schema, before scan projection or
@@ -294,7 +283,6 @@ pub struct FileScanConfigBuilder {
294283
file_compression_type: Option<FileCompressionType>,
295284
batch_size: Option<usize>,
296285
expr_adapter_factory: Option<Arc<dyn PhysicalExprAdapterFactory>>,
297-
partitioned_by_file_group: bool,
298286
}
299287

300288
impl FileScanConfigBuilder {
@@ -321,7 +309,6 @@ impl FileScanConfigBuilder {
321309
constraints: None,
322310
batch_size: None,
323311
expr_adapter_factory: None,
324-
partitioned_by_file_group: false,
325312
}
326313
}
327314

@@ -519,18 +506,6 @@ impl FileScanConfigBuilder {
519506
self
520507
}
521508

522-
/// Set whether file groups are organized by partition column values.
523-
///
524-
/// When set to true, the output partitioning will be declared as Hash partitioning
525-
/// on the partition columns.
526-
pub fn with_partitioned_by_file_group(
527-
mut self,
528-
partitioned_by_file_group: bool,
529-
) -> Self {
530-
self.partitioned_by_file_group = partitioned_by_file_group;
531-
self
532-
}
533-
534509
/// Build the final [`FileScanConfig`] with all the configured settings.
535510
///
536511
/// This method takes ownership of the builder and returns the constructed `FileScanConfig`.
@@ -552,7 +527,6 @@ impl FileScanConfigBuilder {
552527
file_compression_type,
553528
batch_size,
554529
expr_adapter_factory: expr_adapter,
555-
partitioned_by_file_group,
556530
} = self;
557531

558532
let constraints = constraints.unwrap_or_default();
@@ -577,7 +551,6 @@ impl FileScanConfigBuilder {
577551
batch_size,
578552
expr_adapter_factory: expr_adapter,
579553
statistics,
580-
partitioned_by_file_group,
581554
output_partitioning,
582555
}
583556
}
@@ -598,12 +571,15 @@ impl From<FileScanConfig> for FileScanConfigBuilder {
598571
constraints: Some(config.constraints),
599572
batch_size: config.batch_size,
600573
expr_adapter_factory: config.expr_adapter_factory,
601-
partitioned_by_file_group: config.partitioned_by_file_group,
602574
}
603575
}
604576
}
605577

606-
fn hash_partitioning_from_partition_fields(
578+
/// Builds `Partitioning::Hash` over `partition_cols` (resolved to their indices in
579+
/// `schema`) with `partition_count` partitions. Returns `None` when there are no
580+
/// partition columns. Callers use this to declare the output partitioning of a scan
581+
/// whose file groups are organized by partition column values.
582+
pub fn hash_partitioning_from_partition_fields(
607583
schema: &Schema,
608584
partition_cols: &Fields,
609585
partition_count: usize,
@@ -765,7 +741,7 @@ impl DataSource for FileScanConfig {
765741
) -> Result<Option<Arc<dyn DataSource>>> {
766742
// When file groups define output partitioning, repartitioning files
767743
// would invalidate the partition-to-file-group mapping.
768-
if self.output_partitioning.is_some() || self.partitioned_by_file_group {
744+
if self.output_partitioning.is_some() {
769745
return Ok(None);
770746
}
771747

@@ -782,10 +758,8 @@ impl DataSource for FileScanConfig {
782758
/// Returns the output partitioning for this file scan.
783759
///
784760
/// When `output_partitioning` is set, this returns the declared partitioning
785-
/// after applying scan projection. When `partitioned_by_file_group` is true,
786-
/// this returns `Partitioning::Hash` on the Hive partition columns, allowing
787-
/// the optimizer to skip hash repartitioning for aggregates and joins on
788-
/// those columns.
761+
/// after applying scan projection, allowing the optimizer to skip hash
762+
/// repartitioning for aggregates and joins on the partitioning columns.
789763
///
790764
/// If projection or partition count validation fails, this returns
791765
/// `UnknownPartitioning`.
@@ -801,15 +775,7 @@ impl DataSource for FileScanConfig {
801775
/// - Idea: Could allow byte-range splitting within partition-aware groups,
802776
/// preserving I/O parallelism while maintaining partition semantics.
803777
fn output_partitioning(&self) -> Partitioning {
804-
let Some(output_partitioning) = self.output_partitioning.clone().or_else(|| {
805-
self.partitioned_by_file_group.then(|| {
806-
hash_partitioning_from_partition_fields(
807-
self.file_source.table_schema().table_schema(),
808-
self.table_partition_cols(),
809-
self.file_groups.len(),
810-
)
811-
})?
812-
}) else {
778+
let Some(output_partitioning) = self.output_partitioning.clone() else {
813779
return Partitioning::UnknownPartitioning(self.file_groups.len());
814780
};
815781
if output_partitioning.partition_count() != self.file_groups.len() {
@@ -1140,7 +1106,6 @@ impl DataSource for FileScanConfig {
11401106
) -> Option<Arc<dyn Any + Send + Sync>> {
11411107
if self.preserve_order
11421108
|| self.output_partitioning.is_some()
1143-
|| self.partitioned_by_file_group
11441109
|| !config.execution.enable_file_stream_work_stealing
11451110
{
11461111
return None;
@@ -2553,7 +2518,7 @@ mod tests {
25532518
vec![partition_col],
25542519
);
25552520

2556-
// partitioned_by_file_group defaults to false
2521+
// output_partitioning defaults to None
25572522
let partitioning = config.output_partitioning();
25582523
assert!(matches!(partitioning, Partitioning::UnknownPartitioning(_)));
25592524
}
@@ -2613,13 +2578,12 @@ mod tests {
26132578
#[test]
26142579
fn test_output_partitioning_no_partition_columns() {
26152580
let file_schema = aggr_test_schema();
2616-
let mut config = config_for_projection(
2581+
let config = config_for_projection(
26172582
Arc::clone(&file_schema),
26182583
None,
26192584
Statistics::new_unknown(&file_schema),
26202585
vec![], // No partition columns
26212586
);
2622-
config.partitioned_by_file_group = true;
26232587

26242588
let partitioning = config.output_partitioning();
26252589
assert!(matches!(partitioning, Partitioning::UnknownPartitioning(_)));
@@ -2642,12 +2606,16 @@ mod tests {
26422606
Statistics::new_unknown(&file_schema),
26432607
single_partition_col,
26442608
);
2645-
config.partitioned_by_file_group = true;
26462609
config.file_groups = vec![
26472610
FileGroup::new(vec![PartitionedFile::new("f1.parquet".to_string(), 1024)]),
26482611
FileGroup::new(vec![PartitionedFile::new("f2.parquet".to_string(), 1024)]),
26492612
FileGroup::new(vec![PartitionedFile::new("f3.parquet".to_string(), 1024)]),
26502613
];
2614+
config.output_partitioning = hash_partitioning_from_partition_fields(
2615+
config.file_source.table_schema().table_schema(),
2616+
config.table_partition_cols(),
2617+
config.file_groups.len(),
2618+
);
26512619

26522620
let partitioning = config.output_partitioning();
26532621
match partitioning {
@@ -2671,11 +2639,15 @@ mod tests {
26712639
Statistics::new_unknown(&file_schema),
26722640
multiple_partition_cols,
26732641
);
2674-
config.partitioned_by_file_group = true;
26752642
config.file_groups = vec![
26762643
FileGroup::new(vec![PartitionedFile::new("f1.parquet".to_string(), 1024)]),
26772644
FileGroup::new(vec![PartitionedFile::new("f2.parquet".to_string(), 1024)]),
26782645
];
2646+
config.output_partitioning = hash_partitioning_from_partition_fields(
2647+
config.file_source.table_schema().table_schema(),
2648+
config.table_partition_cols(),
2649+
config.file_groups.len(),
2650+
);
26792651

26802652
let partitioning = config.output_partitioning();
26812653
match partitioning {

‎datafusion/datasource/src/file_stream/mod.rs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1630,14 +1630,24 @@ mod tests {
16301630
DataType::Int32,
16311631
false,
16321632
)])));
1633+
// Declaring an output partitioning marks the scan as pre-grouped, which
1634+
// keeps each stream's files local (disables shared work stealing).
1635+
let output_partitioning = self.partitioned_by_file_group.then(|| {
1636+
datafusion_physical_expr::Partitioning::Hash(
1637+
vec![Arc::new(
1638+
datafusion_physical_expr::expressions::Column::new("i", 0),
1639+
)],
1640+
file_groups.len(),
1641+
)
1642+
});
16331643
FileScanConfigBuilder::new(
16341644
ObjectStoreUrl::parse("test:///").unwrap(),
16351645
Arc::new(MockSource::new(table_schema)),
16361646
)
16371647
.with_file_groups(file_groups)
16381648
.with_limit(self.limit)
16391649
.with_preserve_order(self.preserve_order)
1640-
.with_partitioned_by_file_group(self.partitioned_by_file_group)
1650+
.with_output_partitioning(output_partitioning)
16411651
.build()
16421652
}
16431653
}

‎datafusion/proto/src/physical_plan/from_proto.rs‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -586,18 +586,15 @@ pub fn parse_protobuf_file_scan_config(
586586
file_source
587587
};
588588

589-
let mut config_builder = FileScanConfigBuilder::new(object_store_url, file_source)
589+
let config = FileScanConfigBuilder::new(object_store_url, file_source)
590590
.with_file_groups(file_groups)
591591
.with_constraints(constraints)
592592
.with_statistics(statistics)
593593
.with_limit(proto.limit.as_ref().map(|sl| sl.limit as usize))
594594
.with_output_ordering(output_ordering)
595595
.with_output_partitioning(output_partitioning)
596-
.with_batch_size(proto.batch_size.map(|s| s as usize));
597-
if proto.partitioned_by_file_group.unwrap_or(false) {
598-
config_builder = config_builder.with_partitioned_by_file_group(true);
599-
}
600-
let config = config_builder.build();
596+
.with_batch_size(proto.batch_size.map(|s| s as usize))
597+
.build();
601598
Ok(config)
602599
}
603600

‎datafusion/proto/src/physical_plan/to_proto.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -562,7 +562,9 @@ pub fn serialize_file_scan_config(
562562
constraints: Some(conf.constraints.clone().into()),
563563
batch_size: conf.batch_size.map(|s| s as u64),
564564
projection_exprs,
565-
partitioned_by_file_group: Some(conf.partitioned_by_file_group),
565+
// Partition grouping is now encoded in `output_partitioning`; this legacy
566+
// wire field is left unset (readers rely on `output_partitioning`).
567+
partitioned_by_file_group: None,
566568
output_partitioning,
567569
})
568570
}

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4219,24 +4219,6 @@ fn roundtrip_file_scan_config(scan_config: FileScanConfig) -> Result<FileScanCon
42194219
Ok(file_scan_config.clone())
42204220
}
42214221

4222-
#[test]
4223-
fn roundtrip_parquet_exec_partitioned_by_file_group() -> Result<()> {
4224-
let file_schema =
4225-
Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
4226-
let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));
4227-
let scan_config =
4228-
FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
4229-
.with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
4230-
"/path/to/file.parquet".to_string(),
4231-
1024,
4232-
)])])
4233-
.with_partitioned_by_file_group(true)
4234-
.build();
4235-
4236-
assert!(roundtrip_file_scan_config(scan_config)?.partitioned_by_file_group);
4237-
Ok(())
4238-
}
4239-
42404222
#[test]
42414223
fn roundtrip_parquet_exec_output_partitioning() -> Result<()> {
42424224
let file_schema =

0 commit comments

Comments
 (0)