@@ -103,7 +103,7 @@ use datafusion_comet_spark_expr::{
103103 Cast , CreateNamedStruct , DateTruncExpr , GetArrayStructFields , GetStructField , HourExpr , IfExpr ,
104104 ListExtract , MinuteExpr , RLike , SecondExpr , TimestampTruncExpr , ToJson ,
105105} ;
106- use datafusion_common:: config:: TableParquetOptions ;
106+ use datafusion_common:: config:: { ConfigOptions , TableParquetOptions } ;
107107use datafusion_common:: scalar:: ScalarStructBuilder ;
108108use datafusion_common:: {
109109 tree_node:: { Transformed , TransformedResult , TreeNode , TreeNodeRecursion , TreeNodeRewriter } ,
@@ -1030,17 +1030,9 @@ impl PhysicalPlanner {
10301030 . map ( |path| PartitionedFile :: from_path ( path. path ( ) . to_string ( ) ) . unwrap ( ) )
10311031 . collect ( ) ;
10321032
1033- // partition the files
1034- // TODO really should partition the row groups
1035-
1036- let mut file_groups = vec ! [ vec![ ] ; partition_count] ;
1037- files. iter ( ) . enumerate ( ) . for_each ( |( idx, file) | {
1038- file_groups[ idx % partition_count] . push ( file. clone ( ) ) ;
1039- } ) ;
1040-
10411033 let file_scan_config =
10421034 FileScanConfig :: new ( object_store_url, Arc :: clone ( & data_schema_arrow) )
1043- . with_file_groups ( file_groups )
1035+ . with_file_groups ( vec ! [ files ] )
10441036 . with_projection ( Some ( projection_vector) ) ;
10451037
10461038 let mut table_parquet_options = TableParquetOptions :: new ( ) ;
@@ -1054,8 +1046,15 @@ impl PhysicalPlanner {
10541046 builder = builder. with_predicate ( filter) ;
10551047 }
10561048
1049+ let options = ConfigOptions :: default ( ) ;
10571050 let scan = builder. build ( ) ;
1058- return Ok ( ( vec ! [ ] , Arc :: new ( scan) ) ) ;
1051+
1052+ // try and repartition by row group
1053+ if let Some ( partitioned_scan) = scan. repartitioned ( partition_count, & options) ? {
1054+ return Ok ( ( vec ! [ ] , partitioned_scan) ) ;
1055+ } else {
1056+ return Ok ( ( vec ! [ ] , Arc :: new ( scan) ) ) ;
1057+ }
10591058 }
10601059
10611060 // If it is not test execution context for unit test, we should have at least one
0 commit comments