Skip to content

Commit f7850be

Browse files
authored
Merge pull request #3 from andygrove/df-parquet-exec
feat: Repartition files by row group
2 parents e52fe77 + 6441cff commit f7850be

1 file changed

Lines changed: 10 additions & 11 deletions

File tree

‎native/core/src/execution/datafusion/planner.rs‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ use datafusion_comet_spark_expr::{
103103
Cast, CreateNamedStruct, DateTruncExpr, GetArrayStructFields, GetStructField, HourExpr, IfExpr,
104104
ListExtract, MinuteExpr, RLike, SecondExpr, TimestampTruncExpr, ToJson,
105105
};
106-
use datafusion_common::config::TableParquetOptions;
106+
use datafusion_common::config::{ConfigOptions, TableParquetOptions};
107107
use datafusion_common::scalar::ScalarStructBuilder;
108108
use datafusion_common::{
109109
tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter},
@@ -1030,17 +1030,9 @@ impl PhysicalPlanner {
10301030
.map(|path| PartitionedFile::from_path(path.path().to_string()).unwrap())
10311031
.collect();
10321032

1033-
// partition the files
1034-
// TODO really should partition the row groups
1035-
1036-
let mut file_groups = vec![vec![]; partition_count];
1037-
files.iter().enumerate().for_each(|(idx, file)| {
1038-
file_groups[idx % partition_count].push(file.clone());
1039-
});
1040-
10411033
let file_scan_config =
10421034
FileScanConfig::new(object_store_url, Arc::clone(&data_schema_arrow))
1043-
.with_file_groups(file_groups)
1035+
.with_file_groups(vec![files])
10441036
.with_projection(Some(projection_vector));
10451037

10461038
let mut table_parquet_options = TableParquetOptions::new();
@@ -1054,8 +1046,15 @@ impl PhysicalPlanner {
10541046
builder = builder.with_predicate(filter);
10551047
}
10561048

1049+
let options = ConfigOptions::default();
10571050
let scan = builder.build();
1058-
return Ok((vec![], Arc::new(scan)));
1051+
1052+
// try and repartition by row group
1053+
if let Some(partitioned_scan) = scan.repartitioned(partition_count, &options)? {
1054+
return Ok((vec![], partitioned_scan));
1055+
} else {
1056+
return Ok((vec![], Arc::new(scan)));
1057+
}
10591058
}
10601059

10611060
// If it is not test execution context for unit test, we should have at least one

0 commit comments

Comments
 (0)