Skip to content

Commit e2ee2cf

Browse files
committed
refactor: remove redundant partitioned_by_file_group file scan field
`FileScanConfig` had two overlapping ways to declare file scan output partitioning: the `partitioned_by_file_group` bool and `output_partitioning`. Collapse them onto `output_partitioning` as the single source of truth. - Remove the `partitioned_by_file_group` field, the builder field, and the `with_partitioned_by_file_group` builder method. - `ListingTable` now derives the partition-column `Partitioning::Hash` once its file groups are finalized and passes it via `with_output_partitioning`; `hash_partitioning_from_partition_fields` is made `pub` for this. - proto already round-trips `output_partitioning`, so the now-vestigial wire bool is left unset on write and ignored on read (the proto field is kept for backward compatibility). Closes apache#23099. Signed-off-by: Jiawei Zhao <[email protected]>
1 parent 32e27ac commit e2ee2cf

6 files changed

Lines changed: 51 additions & 80 deletions

File tree

‎datafusion/catalog-listing/src/table.rs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@ use datafusion_common::{
3030
};
3131
use datafusion_datasource::file::FileSource;
3232
use datafusion_datasource::file_groups::FileGroup;
33-
use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};
33+
use datafusion_datasource::file_scan_config::{
34+
FileScanConfig, FileScanConfigBuilder, hash_partitioning_from_partition_fields,
35+
};
3436
use datafusion_datasource::file_sink_config::{FileOutputMode, FileSinkConfig};
3537
#[expect(deprecated)]
3638
use datafusion_datasource::schema_adapter::SchemaAdapterFactory;
@@ -628,6 +630,15 @@ impl TableProvider for ListingTable {
628630
);
629631
}
630632
Some(output_partitioning)
633+
} else if partitioned_by_file_group {
634+
// Files are grouped by partition column values: declare Hash
635+
// partitioning on those columns so the optimizer can skip hash
636+
// repartitioning for aggregates and joins on the partition columns.
637+
hash_partitioning_from_partition_fields(
638+
&self.table_schema,
639+
&table_partition_cols.clone().into(),
640+
partitioned_file_lists.len(),
641+
)
631642
} else {
632643
None
633644
};
@@ -650,7 +661,6 @@ impl TableProvider for ListingTable {
650661
.with_output_ordering(output_ordering)
651662
.with_output_partitioning(output_partitioning)
652663
.with_expr_adapter(self.expr_adapter_factory.clone())
653-
.with_partitioned_by_file_group(partitioned_by_file_group)
654664
.build();
655665

656666
// create the execution plan

‎datafusion/datasource/src/file_scan_config/mod.rs‎

Lines changed: 22 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -198,17 +198,6 @@ pub struct FileScanConfig {
198198
/// would be incorrect if there are filters being applied, thus this should be accessed
199199
/// via [`FileScanConfig::statistics`].
200200
pub(crate) statistics: Statistics,
201-
/// When true, file_groups are organized by partition column values
202-
/// and output_partitioning will return Hash partitioning on partition columns.
203-
/// This allows the optimizer to skip hash repartitioning for aggregates and joins
204-
/// on partition columns.
205-
///
206-
/// If the number of file partitions > target_partitions, the file partitions will be grouped
207-
/// in a round-robin fashion such that number of file partitions = target_partitions.
208-
///
209-
/// Follow-up: remove this redundant field in favor of
210-
/// `output_partitioning`, see <https://github.com/apache/datafusion/issues/23099>.
211-
pub partitioned_by_file_group: bool,
212201
/// Declared physical output partitioning for this scan.
213202
///
214203
/// Expressions are against the full table schema, before scan projection or
@@ -288,7 +277,6 @@ pub struct FileScanConfigBuilder {
288277
file_compression_type: Option<FileCompressionType>,
289278
batch_size: Option<usize>,
290279
expr_adapter_factory: Option<Arc<dyn PhysicalExprAdapterFactory>>,
291-
partitioned_by_file_group: bool,
292280
}
293281

294282
impl FileScanConfigBuilder {
@@ -315,7 +303,6 @@ impl FileScanConfigBuilder {
315303
constraints: None,
316304
batch_size: None,
317305
expr_adapter_factory: None,
318-
partitioned_by_file_group: false,
319306
}
320307
}
321308

@@ -513,18 +500,6 @@ impl FileScanConfigBuilder {
513500
self
514501
}
515502

516-
/// Set whether file groups are organized by partition column values.
517-
///
518-
/// When set to true, the output partitioning will be declared as Hash partitioning
519-
/// on the partition columns.
520-
pub fn with_partitioned_by_file_group(
521-
mut self,
522-
partitioned_by_file_group: bool,
523-
) -> Self {
524-
self.partitioned_by_file_group = partitioned_by_file_group;
525-
self
526-
}
527-
528503
/// Build the final [`FileScanConfig`] with all the configured settings.
529504
///
530505
/// This method takes ownership of the builder and returns the constructed `FileScanConfig`.
@@ -546,7 +521,6 @@ impl FileScanConfigBuilder {
546521
file_compression_type,
547522
batch_size,
548523
expr_adapter_factory: expr_adapter,
549-
partitioned_by_file_group,
550524
} = self;
551525

552526
let constraints = constraints.unwrap_or_default();
@@ -571,7 +545,6 @@ impl FileScanConfigBuilder {
571545
batch_size,
572546
expr_adapter_factory: expr_adapter,
573547
statistics,
574-
partitioned_by_file_group,
575548
output_partitioning,
576549
}
577550
}
@@ -592,12 +565,15 @@ impl From<FileScanConfig> for FileScanConfigBuilder {
592565
constraints: Some(config.constraints),
593566
batch_size: config.batch_size,
594567
expr_adapter_factory: config.expr_adapter_factory,
595-
partitioned_by_file_group: config.partitioned_by_file_group,
596568
}
597569
}
598570
}
599571

600-
fn hash_partitioning_from_partition_fields(
572+
/// Builds `Partitioning::Hash` over `partition_cols` (resolved to their indices in
573+
/// `schema`) with `partition_count` partitions. Returns `None` when there are no
574+
/// partition columns. Callers use this to declare the output partitioning of a scan
575+
/// whose file groups are organized by partition column values.
576+
pub fn hash_partitioning_from_partition_fields(
601577
schema: &Schema,
602578
partition_cols: &Fields,
603579
partition_count: usize,
@@ -759,7 +735,7 @@ impl DataSource for FileScanConfig {
759735
) -> Result<Option<Arc<dyn DataSource>>> {
760736
// When file groups define output partitioning, repartitioning files
761737
// would invalidate the partition-to-file-group mapping.
762-
if self.output_partitioning.is_some() || self.partitioned_by_file_group {
738+
if self.output_partitioning.is_some() {
763739
return Ok(None);
764740
}
765741

@@ -776,10 +752,8 @@ impl DataSource for FileScanConfig {
776752
/// Returns the output partitioning for this file scan.
777753
///
778754
/// When `output_partitioning` is set, this returns the declared partitioning
779-
/// after applying scan projection. When `partitioned_by_file_group` is true,
780-
/// this returns `Partitioning::Hash` on the Hive partition columns, allowing
781-
/// the optimizer to skip hash repartitioning for aggregates and joins on
782-
/// those columns.
755+
/// after applying scan projection, allowing the optimizer to skip hash
756+
/// repartitioning for aggregates and joins on the partitioning columns.
783757
///
784758
/// If projection or partition count validation fails, this returns
785759
/// `UnknownPartitioning`.
@@ -795,15 +769,7 @@ impl DataSource for FileScanConfig {
795769
/// - Idea: Could allow byte-range splitting within partition-aware groups,
796770
/// preserving I/O parallelism while maintaining partition semantics.
797771
fn output_partitioning(&self) -> Partitioning {
798-
let Some(output_partitioning) = self.output_partitioning.clone().or_else(|| {
799-
self.partitioned_by_file_group.then(|| {
800-
hash_partitioning_from_partition_fields(
801-
self.file_source.table_schema().table_schema(),
802-
self.table_partition_cols(),
803-
self.file_groups.len(),
804-
)
805-
})?
806-
}) else {
772+
let Some(output_partitioning) = self.output_partitioning.clone() else {
807773
return Partitioning::UnknownPartitioning(self.file_groups.len());
808774
};
809775
if output_partitioning.partition_count() != self.file_groups.len() {
@@ -1127,10 +1093,7 @@ impl DataSource for FileScanConfig {
11271093
/// when file order must be preserved or the file groups define the output
11281094
/// partitioning needed for the rest of the plan
11291095
fn create_sibling_state(&self) -> Option<Arc<dyn Any + Send + Sync>> {
1130-
if self.preserve_order
1131-
|| self.output_partitioning.is_some()
1132-
|| self.partitioned_by_file_group
1133-
{
1096+
if self.preserve_order || self.output_partitioning.is_some() {
11341097
return None;
11351098
}
11361099

@@ -2541,7 +2504,7 @@ mod tests {
25412504
vec![partition_col],
25422505
);
25432506

2544-
// partitioned_by_file_group defaults to false
2507+
// output_partitioning defaults to None
25452508
let partitioning = config.output_partitioning();
25462509
assert!(matches!(partitioning, Partitioning::UnknownPartitioning(_)));
25472510
}
@@ -2601,13 +2564,12 @@ mod tests {
26012564
#[test]
26022565
fn test_output_partitioning_no_partition_columns() {
26032566
let file_schema = aggr_test_schema();
2604-
let mut config = config_for_projection(
2567+
let config = config_for_projection(
26052568
Arc::clone(&file_schema),
26062569
None,
26072570
Statistics::new_unknown(&file_schema),
26082571
vec![], // No partition columns
26092572
);
2610-
config.partitioned_by_file_group = true;
26112573

26122574
let partitioning = config.output_partitioning();
26132575
assert!(matches!(partitioning, Partitioning::UnknownPartitioning(_)));
@@ -2630,12 +2592,16 @@ mod tests {
26302592
Statistics::new_unknown(&file_schema),
26312593
single_partition_col,
26322594
);
2633-
config.partitioned_by_file_group = true;
26342595
config.file_groups = vec![
26352596
FileGroup::new(vec![PartitionedFile::new("f1.parquet".to_string(), 1024)]),
26362597
FileGroup::new(vec![PartitionedFile::new("f2.parquet".to_string(), 1024)]),
26372598
FileGroup::new(vec![PartitionedFile::new("f3.parquet".to_string(), 1024)]),
26382599
];
2600+
config.output_partitioning = hash_partitioning_from_partition_fields(
2601+
config.file_source.table_schema().table_schema(),
2602+
config.table_partition_cols(),
2603+
config.file_groups.len(),
2604+
);
26392605

26402606
let partitioning = config.output_partitioning();
26412607
match partitioning {
@@ -2659,11 +2625,15 @@ mod tests {
26592625
Statistics::new_unknown(&file_schema),
26602626
multiple_partition_cols,
26612627
);
2662-
config.partitioned_by_file_group = true;
26632628
config.file_groups = vec![
26642629
FileGroup::new(vec![PartitionedFile::new("f1.parquet".to_string(), 1024)]),
26652630
FileGroup::new(vec![PartitionedFile::new("f2.parquet".to_string(), 1024)]),
26662631
];
2632+
config.output_partitioning = hash_partitioning_from_partition_fields(
2633+
config.file_source.table_schema().table_schema(),
2634+
config.table_partition_cols(),
2635+
config.file_groups.len(),
2636+
);
26672637

26682638
let partitioning = config.output_partitioning();
26692639
match partitioning {

‎datafusion/datasource/src/file_stream/mod.rs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1581,14 +1581,24 @@ mod tests {
15811581
DataType::Int32,
15821582
false,
15831583
)])));
1584+
// Declaring an output partitioning marks the scan as pre-grouped, which
1585+
// keeps each stream's files local (disables shared work stealing).
1586+
let output_partitioning = self.partitioned_by_file_group.then(|| {
1587+
datafusion_physical_expr::Partitioning::Hash(
1588+
vec![Arc::new(
1589+
datafusion_physical_expr::expressions::Column::new("i", 0),
1590+
)],
1591+
file_groups.len(),
1592+
)
1593+
});
15841594
FileScanConfigBuilder::new(
15851595
ObjectStoreUrl::parse("test:///").unwrap(),
15861596
Arc::new(MockSource::new(table_schema)),
15871597
)
15881598
.with_file_groups(file_groups)
15891599
.with_limit(self.limit)
15901600
.with_preserve_order(self.preserve_order)
1591-
.with_partitioned_by_file_group(self.partitioned_by_file_group)
1601+
.with_output_partitioning(output_partitioning)
15921602
.build()
15931603
}
15941604
}

‎datafusion/proto/src/physical_plan/from_proto.rs‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -586,18 +586,15 @@ pub fn parse_protobuf_file_scan_config(
586586
file_source
587587
};
588588

589-
let mut config_builder = FileScanConfigBuilder::new(object_store_url, file_source)
589+
let config = FileScanConfigBuilder::new(object_store_url, file_source)
590590
.with_file_groups(file_groups)
591591
.with_constraints(constraints)
592592
.with_statistics(statistics)
593593
.with_limit(proto.limit.as_ref().map(|sl| sl.limit as usize))
594594
.with_output_ordering(output_ordering)
595595
.with_output_partitioning(output_partitioning)
596-
.with_batch_size(proto.batch_size.map(|s| s as usize));
597-
if proto.partitioned_by_file_group.unwrap_or(false) {
598-
config_builder = config_builder.with_partitioned_by_file_group(true);
599-
}
600-
let config = config_builder.build();
596+
.with_batch_size(proto.batch_size.map(|s| s as usize))
597+
.build();
601598
Ok(config)
602599
}
603600

‎datafusion/proto/src/physical_plan/to_proto.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -562,7 +562,9 @@ pub fn serialize_file_scan_config(
562562
constraints: Some(conf.constraints.clone().into()),
563563
batch_size: conf.batch_size.map(|s| s as u64),
564564
projection_exprs,
565-
partitioned_by_file_group: Some(conf.partitioned_by_file_group),
565+
// Partition grouping is now encoded in `output_partitioning`; this legacy
566+
// wire field is left unset (readers rely on `output_partitioning`).
567+
partitioned_by_file_group: None,
566568
output_partitioning,
567569
})
568570
}

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4219,24 +4219,6 @@ fn roundtrip_file_scan_config(scan_config: FileScanConfig) -> Result<FileScanCon
42194219
Ok(file_scan_config.clone())
42204220
}
42214221

4222-
#[test]
4223-
fn roundtrip_parquet_exec_partitioned_by_file_group() -> Result<()> {
4224-
let file_schema =
4225-
Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
4226-
let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));
4227-
let scan_config =
4228-
FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
4229-
.with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
4230-
"/path/to/file.parquet".to_string(),
4231-
1024,
4232-
)])])
4233-
.with_partitioned_by_file_group(true)
4234-
.build();
4235-
4236-
assert!(roundtrip_file_scan_config(scan_config)?.partitioned_by_file_group);
4237-
Ok(())
4238-
}
4239-
42404222
#[test]
42414223
fn roundtrip_parquet_exec_output_partitioning() -> Result<()> {
42424224
let file_schema =

0 commit comments

Comments
 (0)